diff --git a/Cargo.lock b/Cargo.lock index 5b9d7ed73..228f4cc62 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6741,6 +6741,7 @@ name = "temper-store-postgres" version = "0.1.0" dependencies = [ "chrono", + "futures", "opentelemetry", "serde", "serde_json", diff --git a/crates/temper-cli/src/migrate_turso_to_postgres.rs b/crates/temper-cli/src/migrate_turso_to_postgres.rs index cf616944d..914f8180c 100644 --- a/crates/temper-cli/src/migrate_turso_to_postgres.rs +++ b/crates/temper-cli/src/migrate_turso_to_postgres.rs @@ -9,7 +9,7 @@ use serde_json::{Value, json}; use sha2::{Digest, Sha256}; use sqlx::{PgPool, Row}; use temper_evolution::PostgresRecordStore; -use temper_runtime::persistence::{EventStore, PersistenceEnvelope}; +use temper_runtime::persistence::{EventStore, PersistenceEnvelope, SnapshotSourceFence}; use temper_store_postgres::PostgresEventStore; use temper_store_turso::{ FeatureRequestRow, TursoEventStore, TursoInstalledAppRow, TursoSpecRow, spec_content_hash, @@ -319,8 +319,26 @@ async fn migrate_event_journal( "state": base64::engine::general_purpose::STANDARD.encode(&snapshot), })); if !dry_run { + let target_source = + match target + .load_snapshot(&persistence_id) + .await + .with_context(|| { + format!("failed to read target snapshot fence for {persistence_id}") + })? { + Some((sequence_nr, state)) => { + SnapshotSourceFence::Exact { sequence_nr, state } + } + None => SnapshotSourceFence::Absent, + }; target - .save_snapshot(&persistence_id, sequence_nr, &snapshot) + .save_snapshot_if_source( + &persistence_id, + sequence_nr, + &snapshot, + &target_source, + None, + ) .await .with_context(|| { format!("failed to write target snapshot for {persistence_id}") diff --git a/crates/temper-cli/src/serve/bootstrap.rs b/crates/temper-cli/src/serve/bootstrap.rs index 2e206e453..ee33c3e0b 100644 --- a/crates/temper-cli/src/serve/bootstrap.rs +++ b/crates/temper-cli/src/serve/bootstrap.rs @@ -3,6 +3,10 @@ //! Each function represents an explicit phase of the startup pipeline. //! The `run` coordinator in `mod.rs` calls these in sequence. +mod hydration; + +pub(super) use hydration::hydrate_entities; + use std::collections::BTreeMap; use std::fs; use std::path::Path; @@ -11,7 +15,6 @@ use std::sync::Arc; use anyhow::{Context, Result}; use temper_platform::state::PlatformState; -use temper_runtime::tenant::TenantId; use temper_server::authz::load_and_activate_tenant_policies; use temper_server::registry::SpecRegistry; use temper_server::registry_bootstrap::{ @@ -211,83 +214,15 @@ pub(super) fn load_webhooks(apps: &[(String, String)]) -> Option = state @@ -385,7 +321,7 @@ pub(super) async fn recover_secrets(state: &PlatformState) { // Collect known tenants from the registry. let tenants: Vec = { - let reg = state.registry.read().unwrap(); // ci-ok: infallible lock + let reg = state.registry.read().expect("registry lock poisoned"); reg.tenant_ids() .into_iter() .map(|t| t.as_str().to_string()) @@ -479,13 +415,6 @@ pub(super) async fn bootstrap_tenants(state: &PlatformState, apps: &[(String, St } } } - - // Auto-register operator credential for the global API key (ADR-0033). - // This ensures the bearer auth middleware resolves the global key as a - // verified "operator" identity instead of falling through as anonymous. - if let Some(ref api_key) = state.api_token { - temper_platform::bootstrap_operator_credential(state, api_key, "default").await; - } } #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -628,7 +557,66 @@ mod tests { use temper_spec::csdl::parse_csdl; use temper_store_turso::TursoEventStore; - use super::bootstrap_installed_apps; + use super::{bootstrap_installed_apps, recover_cedar_policies}; + + const LEGACY_POLICY: &str = r#"permit(principal, action == Action::"legacy_only", resource);"#; + const GRANULAR_POLICY: &str = + r#"permit(principal, action == Action::"granular_only", resource);"#; + + #[tokio::test] + async fn cedar_recovery_prefers_granular_generation_over_legacy_cache() { + let tenant = "bootstrap-policy-migration"; + let bundle = get_os_app("temper-fs").expect("temper-fs app bundle should load"); + let csdl_xml = bundle.csdl.clone().expect("temper-fs should have CSDL"); + let csdl = parse_csdl(&csdl_xml).expect("temper-fs CSDL should parse"); + let spec_refs: Vec<(&str, &str)> = bundle + .specs + .iter() + .map(|(entity_type, source)| (entity_type.as_str(), source.as_str())) + .collect(); + + let mut state = PlatformState::new(None); + state + .registry + .write() + .unwrap() + .register_tenant(tenant, csdl, csdl_xml, &spec_refs); + + let db_path = std::env::temp_dir().join(format!( + "temper-bootstrap-policy-{}.db", + temper_runtime::scheduler::sim_uuid() + )); + let db_url = format!("file:{}", db_path.display()); + let turso = TursoEventStore::new(&db_url, None) + .await + .expect("turso store should initialize"); + turso + .upsert_tenant_policy(tenant, LEGACY_POLICY) + .await + .expect("legacy policy should persist"); + turso + .save_policy(tenant, "decision:new", GRANULAR_POLICY, "test") + .await + .expect("granular policy should persist"); + state + .server + .set_storage_stack(StorageStack::from_turso(turso.clone())); + + recover_cedar_policies(&state).await; + + let active = state + .server + .authz + .get_tenant_policy_text(tenant) + .expect("tenant generation should activate"); + assert!(!active.contains("legacy_only")); + assert!(active.contains("granular_only")); + let rows = turso + .load_policies_for_tenant(tenant) + .await + .expect("canonical policy rows should load"); + assert!(rows.iter().all(|row| row.policy_id != "primary")); + } #[tokio::test] async fn bootstrap_installed_apps_replays_persisted_app_when_registry_specs_are_stale() { diff --git a/crates/temper-cli/src/serve/bootstrap/hydration.rs b/crates/temper-cli/src/serve/bootstrap/hydration.rs new file mode 100644 index 000000000..09d168a56 --- /dev/null +++ b/crates/temper-cli/src/serve/bootstrap/hydration.rs @@ -0,0 +1,149 @@ +//! Startup entity hydration and durable key-contract activation. + +use anyhow::{Context, Result}; +use std::collections::BTreeSet; +use temper_platform::state::PlatformState; +use temper_runtime::tenant::TenantId; + +fn registered_activation_tenants( + all_tenants: &[TenantId], + registered_tenants: &BTreeSet, +) -> Vec { + all_tenants + .iter() + .filter(|tenant| registered_tenants.contains(*tenant)) + .cloned() + .collect() +} + +pub(in crate::serve) async fn hydrate_entities( + state: &PlatformState, + apps: &[(String, String)], +) -> Result<()> { + if state.server.storage_stack.is_none() { + return Ok(()); + } + let eager_hydrate = std::env::var("TEMPER_EAGER_HYDRATE") // determinism-ok: read once at startup + .ok() + .map(|v| { + matches!( + v.trim().to_ascii_lowercase().as_str(), + "1" | "true" | "on" | "yes" + ) + }) + .unwrap_or(false); + let mut all_tenants = Vec::new(); + for (tenant, _dir) in apps { + all_tenants.push(TenantId::new(tenant.as_str())); + } + // In TenantRouted mode, also hydrate all registered tenants. + if let Some(provider) = state + .server + .storage_stack + .as_ref() + .and_then(|stack| stack.turso.clone()) + { + for tenant in provider.connected_tenants().await { + all_tenants.push(TenantId::new(&tenant)); + } + } + // Postgres-restored tenants may have no CLI app directory and no Turso + // router entry. The durable registry is the complete startup authority. + let registered_tenants = state + .server + .registry + .read() + .expect("spec registry lock poisoned") + .tenant_ids() + .into_iter() + .cloned() + .collect::>(); + all_tenants.extend(registered_tenants.iter().cloned()); + if let Some(storage) = state.server.storage_stack.as_ref() { + let activated_contracts = storage + .events + .key_index_activated_contracts() + .await + .map_err(anyhow::Error::msg) + .context("failed to enumerate activated key-contract tenants before serving")?; + all_tenants.extend( + activated_contracts + .into_iter() + .map(|(tenant, _)| TenantId::new(&tenant)), + ); + } + + all_tenants.sort(); + all_tenants.dedup(); + // Establish each loaded spec table's activation epoch before hydration can + // dispatch actors or any network listener can serve writes. A tenant found + // only through durable contract state has no live spec authority yet, so + // activating an empty registry would incorrectly retire its contracts. + for tenant_id in registered_activation_tenants(&all_tenants, ®istered_tenants) { + state + .server + .activate_registered_key_contracts(&tenant_id) + .await + .map_err(anyhow::Error::msg) + .with_context(|| { + format!( + "failed to activate registered key contracts for tenant {tenant_id} before serving" + ) + })?; + } + for tenant_id in &all_tenants { + if eager_hydrate { + state.server.hydrate_from_store(tenant_id).await; + } else { + state.server.populate_index_from_store(tenant_id).await; + } + } + + // Background task: backfill the declared-key index, then the broad field index, + // from snapshots — after the entity index is populated so pre-existing entities + // are covered. + let server = state.server.clone(); + tokio::spawn(async move { + // ADR-0153: the cheap declared-key backfill first (K = 1-3 rows per entity). + // It keys pre-existing entities and sets the per-(tenant,type) watermark, so + // their point reads resolve present/absent in O(log n) instead of the + // full-type scan that 413s at tenant scale — independent of the heavy + // field-index re-scan. No-op on backends that don't co-commit keys (those + // never become authoritative, so a keyed miss stays scan-safe). + for tenant_id in &all_tenants { + server.populate_key_index_from_snapshots(tenant_id).await; + } + // ADR-0155: backfill the declared-vector index (parse + upsert one row per + // declared path per entity), so pre-existing / write-behind entities are + // rankable by Temper.Nearest and the per-type watermark is set. + for tenant_id in &all_tenants { + server.populate_vector_index_from_snapshots(tenant_id).await; + } + // Then the broad field index for OData filter push-down. + for tenant_id in all_tenants { + server.populate_field_index_from_snapshots(&tenant_id).await; + } + }); + Ok(()) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + use temper_runtime::tenant::TenantId; + + use super::registered_activation_tenants; + + #[test] + fn durable_contract_only_tenant_is_not_activated_without_registered_specs() { + let registered = TenantId::new("registered"); + let durable_only = TenantId::new("durable-only"); + let all_tenants = vec![durable_only, registered.clone()]; + let registered_tenants = BTreeSet::from([registered.clone()]); + + assert_eq!( + registered_activation_tenants(&all_tenants, ®istered_tenants), + vec![registered] + ); + } +} diff --git a/crates/temper-cli/src/serve/mod.rs b/crates/temper-cli/src/serve/mod.rs index 8c7616e43..6835645af 100644 --- a/crates/temper-cli/src/serve/mod.rs +++ b/crates/temper-cli/src/serve/mod.rs @@ -137,10 +137,17 @@ pub async fn run( println!(" Secrets vault: configured"); } - // Phase 6: Entity hydration - bootstrap::hydrate_entities(&state, &apps).await; + // Phase 6: Publish every startup spec before activating durable key + // contracts or hydrating actors. The later hydration phase unions the full + // registry, activates each table's monotonic epoch, and only then permits + // entity dispatch. + bootstrap::bootstrap_tenants(&state, &apps).await; + bootstrap::bootstrap_installed_apps(&state, &os_app_installs).await?; + + // Phase 7: Contract activation and entity hydration. + bootstrap::hydrate_entities(&state, &apps).await?; - // Phase 7: Recovery (Cedar policies + WASM modules + secrets) + // Phase 8: Recovery (Cedar policies + WASM modules + secrets) bootstrap::recover_cedar_policies(&state).await; bootstrap::recover_wasm_modules(&state).await; bootstrap::recover_secrets(&state).await; @@ -238,10 +245,11 @@ pub async fn run( println!(); } - // Phase 8: Bootstrap system + agent tenants - bootstrap::bootstrap_tenants(&state, &apps).await; - // Phase 8b: Restore persisted apps + apply CLI `--app` requests. - bootstrap::bootstrap_installed_apps(&state, &os_app_installs).await?; + // Entity-creating bootstrap work runs only after all durable contracts have + // been activated and their epochs attached to the live tables. + if let Some(ref api_key) = state.api_token { + temper_platform::bootstrap_operator_credential(&state, api_key, "default").await; + } if actor_runtime == ActorRuntimeBackend::Postgres { pg_actor_runtime_cancel = Some( actor_runtime::install_postgres_actor_runtime( diff --git a/crates/temper-jit/src/shadow.rs b/crates/temper-jit/src/shadow.rs index 8cd41174f..024129999 100644 --- a/crates/temper-jit/src/shadow.rs +++ b/crates/temper-jit/src/shadow.rs @@ -116,6 +116,7 @@ mod tests { states: vec!["Draft".into(), "Submitted".into(), "Cancelled".into()], initial_state: "Draft".into(), keys: vec![], + key_contract_activation_epoch: 0, vectors: vec![], rules: vec![ TransitionRule { diff --git a/crates/temper-jit/src/swap.rs b/crates/temper-jit/src/swap.rs index 6b05c59f6..3f885bc78 100644 --- a/crates/temper-jit/src/swap.rs +++ b/crates/temper-jit/src/swap.rs @@ -87,6 +87,7 @@ mod tests { states: vec!["A".into(), "B".into()], initial_state: "A".into(), keys: vec![], + key_contract_activation_epoch: 0, vectors: vec![], rules: vec![TransitionRule { name: "GoB".into(), diff --git a/crates/temper-jit/src/table/builder.rs b/crates/temper-jit/src/table/builder.rs index bf48afc67..39f2d00a8 100644 --- a/crates/temper-jit/src/table/builder.rs +++ b/crates/temper-jit/src/table/builder.rs @@ -129,6 +129,7 @@ impl TransitionTable { properties: k.properties.clone(), }) .collect(), + key_contract_activation_epoch: 0, vectors: automaton .vectors .iter() diff --git a/crates/temper-jit/src/table/types.rs b/crates/temper-jit/src/table/types.rs index e18fe1ffe..58e85a4a4 100644 --- a/crates/temper-jit/src/table/types.rs +++ b/crates/temper-jit/src/table/types.rs @@ -53,6 +53,10 @@ pub struct TransitionTable { /// ADR-0153: declared unique/alternate keys the kernel indexes for /// negative-existence reads. Empty when the spec declared no `[[key]]`. pub keys: Vec, + /// Monotonic durable activation epoch for the declared-key contract. + /// Runtime registration fills this after storage fences the spec swap. + #[serde(skip)] + pub key_contract_activation_epoch: u64, /// ADR-0155: declared vector access paths the kernel indexes for exact-scan /// kNN reads (`Temper.Nearest`). Empty when the spec declared no `[[vector]]`. #[serde(default)] @@ -172,6 +176,7 @@ impl<'de> Deserialize<'de> for TransitionTable { initial_state: raw.initial_state, rules: raw.rules, keys: raw.keys, + key_contract_activation_epoch: 0, vectors: raw.vectors, state_var_metadata: raw.state_var_metadata, composite_actions: raw.composite_actions, @@ -287,6 +292,7 @@ mod tests { states: vec!["Draft".to_string(), "Active".to_string()], initial_state: "Draft".to_string(), keys: vec![], + key_contract_activation_epoch: 0, vectors: vec![], rules: vec![ TransitionRule { diff --git a/crates/temper-platform/src/bootstrap.rs b/crates/temper-platform/src/bootstrap.rs index 8cbbdd176..4590d24d9 100644 --- a/crates/temper-platform/src/bootstrap.rs +++ b/crates/temper-platform/src/bootstrap.rs @@ -93,6 +93,7 @@ pub(crate) struct BootstrapTenantSpecsOptions<'a> { pub(crate) verified_cache: &'a BTreeMap, pub(crate) cross_invariants_source: Option<&'a str>, pub(crate) verification_mode: BootstrapSpecVerificationMode, + pub(crate) key_contract_activation_epochs: Option<&'a BTreeMap>, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -129,6 +130,7 @@ fn bootstrap_tenant_specs_inner( verified_cache, cross_invariants_source, verification_mode, + key_contract_activation_epochs, } = options; tracing::info!( @@ -186,8 +188,9 @@ fn bootstrap_tenant_specs_inner( let tenant_id = TenantId::new(tenant); { let mut registry = state.registry.write().unwrap(); // ci-ok: infallible lock + let empty_key_contract_epochs = BTreeMap::new(); registry - .try_register_tenant_with_reactions_and_constraints( + .try_register_tenant_with_reactions_constraints_and_key_epochs( tenant_id.clone(), csdl, csdl_source.to_string(), @@ -195,6 +198,7 @@ fn bootstrap_tenant_specs_inner( Vec::new(), cross_invariants_source.map(str::to_string), merge, + key_contract_activation_epochs.unwrap_or(&empty_key_contract_epochs), ) .unwrap_or_else(|e| panic!("failed to register {label} specs for '{tenant}': {e}")); let now = temper_runtime::scheduler::sim_now().to_rfc3339(); @@ -244,6 +248,7 @@ pub fn bootstrap_system_tenant( verified_cache, cross_invariants_source: None, verification_mode: BootstrapSpecVerificationMode::FullCascade, + key_contract_activation_epochs: None, }, ) } @@ -270,6 +275,7 @@ pub fn bootstrap_agent_specs( verified_cache, cross_invariants_source: None, verification_mode: BootstrapSpecVerificationMode::FullCascade, + key_contract_activation_epochs: None, }, ) } diff --git a/crates/temper-platform/src/deploy.rs b/crates/temper-platform/src/deploy.rs index 533f83ada..6db08df12 100644 --- a/crates/temper-platform/src/deploy.rs +++ b/crates/temper-platform/src/deploy.rs @@ -9,6 +9,8 @@ //! └─ temper.verify.{Entity} (cascade_passed, l1, l2, l3) //! ``` +mod publication; + use opentelemetry::KeyValue; use opentelemetry::global; use opentelemetry::trace::{Span, Status, Tracer}; @@ -82,7 +84,7 @@ impl DeployPipeline { /// 5. Broadcast deployment status /// /// Emits a parent `temper.deploy` span with child spans per entity. - pub fn verify_and_deploy(state: &PlatformState, input: &DeployInput) -> DeployResult { + pub async fn verify_and_deploy(state: &PlatformState, input: &DeployInput) -> DeployResult { let tracer = global::tracer("temper"); let mut deploy_span = tracer .span_builder("temper.deploy") @@ -293,22 +295,13 @@ impl DeployPipeline { if all_passed && !input.entities.is_empty() { match parse_csdl(&input.csdl_xml) { Ok(csdl) => { - // Collect IOA sources for registration - let ioa_pairs: Vec<(&str, &str)> = entity_results - .iter() - .map(|r| (r.entity_name.as_str(), r.ioa_source.as_str())) - .collect(); - - // Register tenant in the live registry. - let register_result = { - let mut registry = state.registry.write().unwrap(); - registry.try_register_tenant( - TenantId::new(&input.tenant_name), - csdl, - input.csdl_xml.clone(), - &ioa_pairs, - ) - }; + let register_result = publication::publish_verified_generation( + state, + input, + &entity_results, + csdl, + ) + .await; match register_result { Ok(()) => { @@ -508,12 +501,12 @@ kind = "internal" } } - #[test] - fn test_deploy_pipeline_success() { + #[tokio::test] + async fn test_deploy_pipeline_success() { let state = PlatformState::new(None); let mut rx = state.subscribe(); - let result = DeployPipeline::verify_and_deploy(&state, &sample_deploy_input()); + let result = DeployPipeline::verify_and_deploy(&state, &sample_deploy_input()).await; assert!( result.success, @@ -532,11 +525,11 @@ kind = "internal" assert!(!received.is_empty(), "Should have broadcast messages"); } - #[test] - fn test_deploy_pipeline_registers_tenant() { + #[tokio::test] + async fn test_deploy_pipeline_registers_tenant() { let state = PlatformState::new(None); - let result = DeployPipeline::verify_and_deploy(&state, &sample_deploy_input()); + let result = DeployPipeline::verify_and_deploy(&state, &sample_deploy_input()).await; assert!(result.success); @@ -547,8 +540,8 @@ kind = "internal" assert!(registry.get_table(&tenant, "Task").is_some()); } - #[test] - fn test_deploy_pipeline_empty_entities() { + #[tokio::test] + async fn test_deploy_pipeline_empty_entities() { let state = PlatformState::new(None); let input = DeployInput { @@ -557,18 +550,18 @@ kind = "internal" entities: vec![], wasm_modules: std::collections::BTreeMap::new(), }; - let result = DeployPipeline::verify_and_deploy(&state, &input); + let result = DeployPipeline::verify_and_deploy(&state, &input).await; // Empty entities should succeed vacuously assert!(result.success); assert!(result.entity_results.is_empty()); } - #[test] - fn test_deploy_pipeline_verification_results() { + #[tokio::test] + async fn test_deploy_pipeline_verification_results() { let state = PlatformState::new(None); - let result = DeployPipeline::verify_and_deploy(&state, &sample_deploy_input()); + let result = DeployPipeline::verify_and_deploy(&state, &sample_deploy_input()).await; assert!(result.success); let entity_result = &result.entity_results[0]; @@ -577,12 +570,12 @@ kind = "internal" assert!(cascade.all_passed); } - #[test] - fn test_deploy_pipeline_broadcasts_verify_status() { + #[tokio::test] + async fn test_deploy_pipeline_broadcasts_verify_status() { let state = PlatformState::new(None); let mut rx = state.subscribe(); - let _result = DeployPipeline::verify_and_deploy(&state, &sample_deploy_input()); + let _result = DeployPipeline::verify_and_deploy(&state, &sample_deploy_input()).await; let mut verify_msgs = Vec::new(); let mut deploy_msgs = Vec::new(); @@ -605,21 +598,21 @@ kind = "internal" ); } - #[test] - fn test_deploy_result_summary() { + #[tokio::test] + async fn test_deploy_result_summary() { let state = PlatformState::new(None); - let result = DeployPipeline::verify_and_deploy(&state, &sample_deploy_input()); + let result = DeployPipeline::verify_and_deploy(&state, &sample_deploy_input()).await; assert!(result.summary.contains("Successfully deployed")); assert!(result.summary.contains("test-tenant")); } - #[test] - fn test_deploy_pipeline_span_noop() { + #[tokio::test] + async fn test_deploy_pipeline_span_noop() { // Verifies that OTEL span instrumentation doesn't panic with no-op tracer. let state = PlatformState::new(None); - let result = DeployPipeline::verify_and_deploy(&state, &sample_deploy_input()); + let result = DeployPipeline::verify_and_deploy(&state, &sample_deploy_input()).await; assert!( result.success, "Pipeline should succeed with no-op OTEL: {}", @@ -627,8 +620,8 @@ kind = "internal" ); } - #[test] - fn test_deploy_multiple_entities() { + #[tokio::test] + async fn test_deploy_multiple_entities() { let state = PlatformState::new(None); let input = DeployInput { @@ -647,7 +640,7 @@ kind = "internal" wasm_modules: std::collections::BTreeMap::new(), }; - let result = DeployPipeline::verify_and_deploy(&state, &input); + let result = DeployPipeline::verify_and_deploy(&state, &input).await; assert!( result.success, @@ -662,8 +655,8 @@ kind = "internal" assert!(registry.get_table(&tenant, "Bug").is_some()); } - #[test] - fn test_deploy_bad_ioa_fails() { + #[tokio::test] + async fn test_deploy_bad_ioa_fails() { let state = PlatformState::new(None); let input = DeployInput { @@ -676,7 +669,7 @@ kind = "internal" wasm_modules: std::collections::BTreeMap::new(), }; - let result = DeployPipeline::verify_and_deploy(&state, &input); + let result = DeployPipeline::verify_and_deploy(&state, &input).await; assert!(!result.success); assert!(!result.entity_results[0].verified); } diff --git a/crates/temper-platform/src/deploy/publication.rs b/crates/temper-platform/src/deploy/publication.rs new file mode 100644 index 000000000..1757d3b73 --- /dev/null +++ b/crates/temper-platform/src/deploy/publication.rs @@ -0,0 +1,222 @@ +//! Durable publication and runtime cutover for verified deploy generations. + +use std::collections::{BTreeMap, BTreeSet}; + +use temper_runtime::tenant::TenantId; +use temper_server::platform_store::{ + SpecPublication, SpecPublicationMode, TenantConstraintsPublication, TenantPolicyPublication, + WasmPublication, +}; +use temper_spec::csdl::CsdlDocument; +use temper_store_turso::spec_content_hash; + +use super::{DeployInput, EntityDeployResult}; +use crate::state::PlatformState; + +async fn activate_deploy_wasm_modules( + state: &PlatformState, + input: &DeployInput, +) -> Result<(), String> { + let tenant_id = TenantId::new(&input.tenant_name); + for (module_name, wasm_bytes) in &input.wasm_modules { + let hash = temper_wasm::WasmEngine::hash_module(wasm_bytes); + state + .server + .wasm_module_registry + .write() + .expect("WASM module registry lock poisoned") + .register(&tenant_id, module_name, &hash); + } + Ok(()) +} + +fn validate_deploy_wasm_modules(state: &PlatformState, input: &DeployInput) -> Result<(), String> { + for (module_name, wasm_bytes) in &input.wasm_modules { + state + .server + .wasm_engine + .compile_and_cache(wasm_bytes) + .map_err(|error| format!("invalid deploy WASM module '{module_name}': {error}"))?; + } + Ok(()) +} + +pub(super) async fn publish_verified_generation( + state: &PlatformState, + input: &DeployInput, + entity_results: &[EntityDeployResult], + csdl: CsdlDocument, +) -> Result<(), String> { + validate_deploy_wasm_modules(state, input)?; + let tenant_id = TenantId::new(&input.tenant_name); + let ioa_pairs = entity_results + .iter() + .map(|result| (result.entity_name.as_str(), result.ioa_source.as_str())) + .collect::>(); + let mut publication_guard = state.server.begin_spec_publication(&tenant_id).await?; + let preserved_constraints = state + .registry + .read() + .expect("registry lock poisoned") + .get_tenant(&tenant_id) + .and_then(|config| config.cross_invariants_source.clone()); + + let mut entity_types = BTreeSet::new(); + for entity in entity_results { + if !entity_types.insert(entity.entity_name.as_str()) { + return Err(format!( + "duplicate entity type {} in deploy generation", + entity.entity_name + )); + } + } + let mut candidate_registry = temper_server::registry::SpecRegistry::new(); + candidate_registry + .try_register_tenant_with_reactions_constraints_and_key_epochs( + tenant_id.clone(), + csdl.clone(), + input.csdl_xml.clone(), + &ioa_pairs, + Vec::new(), + preserved_constraints.clone(), + false, + &BTreeMap::new(), + ) + .map_err(|error| format!("invalid complete deploy generation: {error}"))?; + + let incoming_types = entity_results + .iter() + .map(|result| result.entity_name.as_str()) + .collect::>(); + let mut removed_entity_types = state + .registry + .read() + .expect("registry lock poisoned") + .entity_types(&tenant_id) + .into_iter() + .filter(|entity_type| !incoming_types.contains(entity_type)) + .map(str::to_string) + .collect::>(); + + let publications = entity_results + .iter() + .map(|entity| (entity, spec_content_hash(&entity.ioa_source))) + .collect::>(); + let publication_refs = publications + .iter() + .map(|(entity, content_hash)| SpecPublication { + entity_type: &entity.entity_name, + ioa_source: &entity.ioa_source, + csdl_xml: &input.csdl_xml, + content_hash, + }) + .collect::>(); + let wasm_publications = input + .wasm_modules + .iter() + .map(|(module_name, wasm_bytes)| { + ( + module_name, + wasm_bytes, + temper_wasm::WasmEngine::hash_module(wasm_bytes), + ) + }) + .collect::>(); + let wasm_publication_refs = wasm_publications + .iter() + .map(|(module_name, wasm_bytes, hash)| WasmPublication { + module_name, + wasm_bytes, + sha256_hash: hash, + source: "upload", + }) + .collect::>(); + + let mut intent_components = vec![("csdl".to_string(), input.csdl_xml.as_bytes().to_vec())]; + intent_components.push(( + "constraints".to_string(), + preserved_constraints + .as_deref() + .unwrap_or("") + .as_bytes() + .to_vec(), + )); + intent_components.extend(entity_results.iter().map(|entity| { + ( + format!("spec:{}", entity.entity_name), + entity.ioa_source.as_bytes().to_vec(), + ) + })); + intent_components.extend( + input + .wasm_modules + .iter() + .map(|(name, bytes)| (format!("wasm:{name}"), bytes.clone())), + ); + let intent_refs = intent_components + .iter() + .map(|(name, value)| (name.as_str(), value.as_slice())) + .collect::>(); + let publication_intent = + temper_server::ServerState::spec_publication_intent("platform-deploy-replace", intent_refs); + state + .server + .arm_spec_publication(&mut publication_guard, &tenant_id, &publication_intent)?; + + if let Some(store) = state + .server + .storage_stack + .as_ref() + .and_then(|stack| stack.platform.clone()) + { + let durable_removed = store + .publish_specs( + &input.tenant_name, + &publication_refs, + SpecPublicationMode::Replace, + TenantConstraintsPublication::Preserve, + TenantPolicyPublication::Preserve, + None, + None, + &wasm_publication_refs, + ) + .await + .map_err(|error| format!("durable spec publication failed: {error}"))?; + removed_entity_types.extend(durable_removed); + } + let removed_entity_types = removed_entity_types.into_iter().collect::>(); + let mut cutover = state + .server + .prepare_key_index_contracts_for_spec_activation_with_removals( + &publication_guard, + &tenant_id, + &ioa_pairs, + &removed_entity_types, + ) + .await?; + + state + .registry + .write() + .expect("registry lock poisoned") + .try_register_tenant_with_reactions_constraints_and_key_epochs( + tenant_id.clone(), + csdl, + input.csdl_xml.clone(), + &ioa_pairs, + Vec::new(), + preserved_constraints, + false, + &cutover.activation_epochs, + ) + .map_err(|error| error.to_string())?; + state + .server + .finish_key_index_contract_activation(&mut publication_guard, &tenant_id, &mut cutover) + .await?; + state.server.rebuild_reaction_dispatcher(); + activate_deploy_wasm_modules(state, input).await?; + state + .server + .complete_spec_publication_retry(&mut publication_guard, &tenant_id) +} diff --git a/crates/temper-platform/src/genesis_install.rs b/crates/temper-platform/src/genesis_install.rs index 3cf880ba0..7e877e7bf 100644 --- a/crates/temper-platform/src/genesis_install.rs +++ b/crates/temper-platform/src/genesis_install.rs @@ -4,7 +4,7 @@ //! governed action has succeeded, then materializes the pinned Genesis commit //! into the platform's app installer. -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use std::path::{Component, Path, PathBuf}; use std::time::Instant; @@ -17,7 +17,7 @@ use temper_server::state::{BoundActionHook, BoundActionHookContext, DispatchComm use crate::os_apps::{ AppManifest, InstallResult, OsAppReconcileResult, add_os_apps_dir_preferred, - os_app_bundle_digest, reconcile_os_app, resolve_os_app_install_order, + os_app_bundle_digest, reconcile_os_app_under_guard, resolve_os_app_install_order, }; use crate::state::PlatformState; @@ -504,27 +504,19 @@ pub async fn install_genesis_app_from_registry( add_os_apps_dir_preferred(cache_root.clone()); let install_platform = platform.clone(); - let reconcile_started = Instant::now(); - let install = - reconcile_materialized_app_closure(&install_platform, &request.tenant, &root_ref.name) - .await?; - log_genesis_install_phase( - &request.app_ref, - "install_reconcile", - reconcile_started, - materialized.len(), - install.wasm_modules.len(), - ); let root_closure_id = format!( "genesis:{}:{}", request.app_ref, root_hash.trim_start_matches('@') ); - + let mut publication_records = BTreeMap::new(); for materialized_ref in &materialized_refs { - let Some(version_hash) = materialized_ref.version_hash.as_deref() else { - continue; - }; + let version_hash = materialized_ref.version_hash.as_deref().ok_or_else(|| { + format!( + "materialized Genesis app {}/{} has no pinned version", + materialized_ref.owner, materialized_ref.name + ) + })?; let app_ref = format!( "{}/{}@{}", materialized_ref.owner, @@ -541,7 +533,7 @@ pub async fn install_genesis_app_from_registry( version_hash.trim_start_matches('@') ) }; - record_genesis_install_metadata( + let record = genesis_install_record( &install_platform, GenesisInstallMetadata { target_tenant: &request.tenant, @@ -554,8 +546,25 @@ pub async fn install_genesis_app_from_registry( follow_policy: &follow_policy, }, ) - .await; + .await?; + publication_records.insert(materialized_ref.name.clone(), record); } + let reconcile_started = Instant::now(); + let install = reconcile_materialized_app_closure( + &install_platform, + &request.tenant, + &root_ref.name, + None, + publication_records, + ) + .await?; + log_genesis_install_phase( + &request.app_ref, + "install_reconcile", + reconcile_started, + materialized.len(), + install.wasm_modules.len(), + ); log_genesis_install_phase( &request.app_ref, "total", @@ -588,13 +597,33 @@ async fn reconcile_materialized_app_closure( platform: &PlatformState, tenant: &str, root_app_name: &str, + expected_generation: Option, + mut publication_records: BTreeMap, ) -> Result { let order = resolve_os_app_install_order(&[root_app_name.to_string()])?; let mut root_result = InstallResult::default(); + let tenant_id = TenantId::new(tenant); + let mut publication_guard = match expected_generation { + Some(expected_generation) => { + platform + .server + .begin_spec_publication_after_drain(&tenant_id, expected_generation) + .await? + } + None => platform.server.begin_spec_publication(&tenant_id).await?, + }; for app_name in order { let started = Instant::now(); - match reconcile_os_app(platform, tenant, &app_name).await? { + match reconcile_os_app_under_guard( + platform, + tenant, + &app_name, + publication_records.remove(&app_name), + &mut publication_guard, + ) + .await? + { OsAppReconcileResult::Skipped { bundle_digest, .. } => { tracing::info!( tenant = %tenant, @@ -621,6 +650,13 @@ async fn reconcile_materialized_app_closure( } } + if !publication_records.is_empty() { + return Err(format!( + "Genesis publication records were not part of the resolved dependency closure: {:?}", + publication_records.keys().collect::>() + )); + } + Ok(root_result) } @@ -1027,6 +1063,10 @@ async fn run_git(cwd: Option<&Path>, args: &[&str]) -> Result { #[async_trait::async_trait] impl BoundActionHook for GenesisInstallHook { + fn requires_generation_handoff(&self, entity_type: &str, action: &str) -> bool { + entity_type == "App" && action.rsplit('.').next().unwrap_or(action) == "Install" + } + async fn after_bound_action( &self, ctx: BoundActionHookContext<'_>, @@ -1039,6 +1079,8 @@ impl BoundActionHook for GenesisInstallHook { action, params, state_json, + operation_id, + expected_generation, } = ctx; if entity_type != "App" || action.rsplit('.').next().unwrap_or(action) != "Install" { @@ -1077,9 +1119,20 @@ impl BoundActionHook for GenesisInstallHook { .unwrap_or_default(), )?; let installation_id = installation_id(entity_id, &target_tenant, &version_hash); + if let Ok(existing) = state + .get_tenant_entity_state(tenant, "AppInstallation", &installation_id) + .await + { + let existing_fields = &existing.state.fields; + if string_field(existing_fields, "HookOperationId").as_deref() == Some(operation_id) + && let Some(output) = existing_fields.get("HookOutput") + { + return Ok(Some(output.clone())); + } + } let cache_root = genesis_cache_root(state, &app_ref); - let materialized_apps = materialize_app_closure( + let materialized_bundles = materialize_app_closure( state, tenant, &cache_root, @@ -1096,27 +1149,81 @@ impl BoundActionHook for GenesisInstallHook { let mut platform = self.platform.clone(); platform.server = state.clone(); - match reconcile_materialized_app_closure(&platform, &target_tenant, &name).await { - Ok(result) => { - let closure_id = format!( + let closure_id = format!( + "genesis:{}:{}", + app_ref, + version_hash.trim_start_matches('@') + ); + let mut publication_records = BTreeMap::new(); + for materialized in &materialized_bundles { + let materialized_app_ref = format!( + "{}/{}@{}", + materialized.owner, + materialized.name, + materialized.version_hash.trim_start_matches('@') + ); + let materialized_closure_id = if materialized.name == name { + closure_id.clone() + } else { + format!( "genesis:{}:{}", - app_ref, - version_hash.trim_start_matches('@') - ); - record_genesis_install_metadata( - &platform, - GenesisInstallMetadata { - target_tenant: &target_tenant, - app_name: &name, - app_ref: &app_ref, - version_hash: &version_hash, - closure_id: &closure_id, - registry_url: ®istry_url, - registry_tenant: ®istry_tenant, - follow_policy: &follow_policy, - }, + materialized_app_ref, + materialized.version_hash.trim_start_matches('@') ) - .await; + }; + let record = genesis_install_record( + &platform, + GenesisInstallMetadata { + target_tenant: &target_tenant, + app_name: &materialized.name, + app_ref: &materialized_app_ref, + version_hash: &materialized.version_hash, + closure_id: &materialized_closure_id, + registry_url: ®istry_url, + registry_tenant: ®istry_tenant, + follow_policy: &follow_policy, + }, + ) + .await?; + publication_records.insert(materialized.name.clone(), record); + } + let materialized_apps = materialized_bundles + .iter() + .map(|materialized| materialized.name.clone()) + .collect::>(); + let expected_generation = (target_tenant == tenant.as_str()) + .then_some(expected_generation) + .flatten(); + match reconcile_materialized_app_closure( + &platform, + &target_tenant, + &name, + expected_generation, + publication_records, + ) + .await + { + Ok(result) => { + let added_count = result.added.len(); + let updated_count = result.updated.len(); + let skipped_count = result.skipped.len(); + let output = serde_json::json!({ + "kind": "genesis_app_install", + "appRef": app_ref.clone(), + "targetTenant": target_tenant.clone(), + "followPolicy": follow_policy.clone(), + "installationId": installation_id.clone(), + "materializedPath": app_dir.clone(), + "materializedApps": materialized_apps.clone(), + "added": result.added, + "updated": result.updated, + "skipped": result.skipped, + "wasmModules": result.wasm_modules, + "agents": result.agents, + "agentSkills": result.skills, + "adrs": result.adrs_bootstrapped, + "seedInstances": result.seed_instances, + }); mark_installation( state, tenant, @@ -1128,45 +1235,36 @@ impl BoundActionHook for GenesisInstallHook { "Installed {} into {} ({} added, {} updated, {} skipped)", app_ref, target_tenant, - result.added.len(), - result.updated.len(), - result.skipped.len() + added_count, + updated_count, + skipped_count ), - "InstalledAt": chrono::Utc::now().to_rfc3339(), + "InstalledAt": temper_runtime::scheduler::sim_now().to_rfc3339(), + "HookOperationId": operation_id, + "HookOutput": output.clone(), }), ) - .await; - Ok(Some(serde_json::json!({ - "kind": "genesis_app_install", - "appRef": app_ref, - "targetTenant": target_tenant, - "followPolicy": follow_policy, - "installationId": installation_id, - "materializedPath": app_dir, - "materializedApps": materialized_apps, - "added": result.added, - "updated": result.updated, - "skipped": result.skipped, - "wasmModules": result.wasm_modules, - "agents": result.agents, - "agentSkills": result.skills, - "adrs": result.adrs_bootstrapped, - "seedInstances": result.seed_instances, - }))) + .await?; + Ok(Some(output)) } Err(error) => { let message = error.to_string(); - mark_installation( + let mark_result = mark_installation( state, tenant, &installation_id, "MarkFailed", serde_json::json!({ "Message": message, - "InstalledAt": chrono::Utc::now().to_rfc3339(), + "InstalledAt": temper_runtime::scheduler::sim_now().to_rfc3339(), }), ) .await; + if let Err(mark_error) = mark_result { + return Err(format!( + "Genesis App.Install failed for {app_ref}: {error}; failed to record installation failure: {mark_error}" + )); + } Err(format!("Genesis App.Install failed for {app_ref}: {error}")) } } @@ -1239,43 +1337,33 @@ fn normalize_follow_policy(raw: &str) -> Result { )) } -async fn record_genesis_install_metadata( +async fn genesis_install_record( platform: &PlatformState, metadata: GenesisInstallMetadata<'_>, -) { - let Some(ps) = platform +) -> Result { + let ps = platform .server .storage_stack .as_ref() - .and_then(|stack| stack.platform.clone()) - else { - return; - }; - let Some(digest) = os_app_bundle_digest(metadata.app_name) else { - tracing::warn!( - tenant = %metadata.target_tenant, - app = %metadata.app_name, - app_ref = %metadata.app_ref, - "Installed Genesis app but could not compute bundle digest for durable provenance" - ); - return; - }; + .and_then(|stack| stack.platform.clone()); + let digest = os_app_bundle_digest(metadata.app_name).ok_or_else(|| { + format!( + "could not compute bundle digest for Genesis app provenance {}:{}", + metadata.target_tenant, metadata.app_name + ) + })?; - let existing_record = match ps - .get_installed_app(metadata.target_tenant, metadata.app_name) - .await - { - Ok(record) => record, - Err(error) => { - tracing::warn!( - tenant = %metadata.target_tenant, - app = %metadata.app_name, - app_ref = %metadata.app_ref, - error = %error, - "Failed to read existing Genesis app provenance before update" - ); - None - } + let existing_record = match ps { + Some(store) => store + .get_installed_app(metadata.target_tenant, metadata.app_name) + .await + .map_err(|error| { + format!( + "failed to read existing Genesis provenance for {}:{}: {error}", + metadata.target_tenant, metadata.app_name + ) + })?, + None => None, }; let (pinned_version_hash, current_version_hash) = provenance_hashes_for_policy( metadata.follow_policy, @@ -1283,7 +1371,7 @@ async fn record_genesis_install_metadata( existing_record.as_ref(), ); - let record = InstalledAppRecord { + Ok(InstalledAppRecord { tenant: metadata.target_tenant.to_string(), app_name: digest.app_name, source_kind: "genesis".to_string(), @@ -1305,17 +1393,7 @@ async fn record_genesis_install_metadata( installed_at: None, last_reconciled_at: None, status: "installed".to_string(), - }; - - if let Err(error) = ps.record_installed_app_metadata(&record).await { - tracing::warn!( - tenant = %metadata.target_tenant, - app = %metadata.app_name, - app_ref = %metadata.app_ref, - error = %error, - "Failed to persist Genesis app provenance" - ); - } + }) } fn provenance_hashes_for_policy( @@ -1561,7 +1639,7 @@ async fn materialize_app_closure( tenant: &TenantId, cache_root: &Path, root: GenesisAppBundle, -) -> Result, String> { +) -> Result, String> { let mut stack = vec![root]; let mut seen = BTreeSet::new(); let mut materialized = Vec::new(); @@ -1580,7 +1658,7 @@ async fn materialize_app_closure( &app_dir, ) .await?; - materialized.push(app.name.clone()); + materialized.push(app.clone()); for dependency in read_manifest_dependencies(&app_dir)?.into_iter().rev() { let dependency = resolve_genesis_dependency(state, tenant, &app.owner, &dependency) @@ -1712,9 +1790,9 @@ async fn mark_installation( installation_id: &str, action: &str, params: Value, -) { +) -> Result<(), String> { let agent_ctx = temper_server::request_context::AgentContext::for_service("genesis-install"); - let _ = state + let response = state .dispatch(DispatchCommand { tenant, entity_type: "AppInstallation", @@ -1725,7 +1803,14 @@ async fn mark_installation( await_integration: false, await_reactions: true, }) - .await; + .await + .map_err(|error| error.to_string())?; + if !response.success { + return Err(response + .error + .unwrap_or_else(|| format!("AppInstallation.{action} failed"))); + } + Ok(()) } async fn materialize_commit_tree( diff --git a/crates/temper-platform/src/hooks.rs b/crates/temper-platform/src/hooks.rs index 6e55d6ded..4c79b37b4 100644 --- a/crates/temper-platform/src/hooks.rs +++ b/crates/temper-platform/src/hooks.rs @@ -28,7 +28,7 @@ mod governance_callback; /// /// Returns `Ok(())` if the hook ran successfully or the effect was /// unrecognized (silently ignored). Returns `Err` if the hook failed. -pub fn dispatch_custom_effect( +pub async fn dispatch_custom_effect( effect_name: &str, entity_type: &str, entity_id: &str, @@ -36,9 +36,9 @@ pub fn dispatch_custom_effect( state: &PlatformState, ) -> Result<(), String> { match effect_name { - "DeploySpecs" => handle_deploy_specs(entity_type, entity_id, state), + "DeploySpecs" => handle_deploy_specs(entity_type, entity_id, state).await, "GenerateCedarPolicy" => { - handle_generate_cedar_policy(entity_type, entity_id, _params, state) + handle_generate_cedar_policy(entity_type, entity_id, _params, state).await } _ => { tracing::debug!( @@ -56,7 +56,7 @@ pub fn dispatch_custom_effect( /// /// Reads specs from the [`SpecStore`], builds a [`DeployInput`], and runs /// the verify-and-deploy pipeline. On success, removes specs from the store. -fn handle_deploy_specs( +async fn handle_deploy_specs( _entity_type: &str, entity_id: &str, state: &PlatformState, @@ -100,7 +100,7 @@ fn handle_deploy_specs( }; // Run the verify-and-deploy pipeline. - let result = DeployPipeline::verify_and_deploy(state, &input); + let result = DeployPipeline::verify_and_deploy(state, &input).await; if result.success { tracing::info!(tenant = entity_id, "DeploySpecs hook: pipeline succeeded"); @@ -133,124 +133,13 @@ fn handle_deploy_specs( /// Reads the entity's fields from the action params, generates a Cedar /// permit statement based on the scope, validates the combined policy set, /// and reloads the authz engine. -fn handle_generate_cedar_policy( +async fn handle_generate_cedar_policy( _entity_type: &str, entity_id: &str, params: &serde_json::Value, state: &PlatformState, ) -> Result<(), String> { - tracing::info!( - entity_id = entity_id, - "GenerateCedarPolicy hook: generating Cedar policy from GovernanceDecision" - ); - - let agent_id = params - .get("agent_id") - .and_then(|v| v.as_str()) - .unwrap_or(""); - let action_name = params - .get("action_name") - .and_then(|v| v.as_str()) - .unwrap_or(""); - let resource_type = params - .get("resource_type") - .and_then(|v| v.as_str()) - .unwrap_or(""); - let resource_id = params - .get("resource_id") - .and_then(|v| v.as_str()) - .unwrap_or(""); - let scope = params - .get("scope") - .and_then(|v| v.as_str()) - .unwrap_or("narrow"); - let tenant = params.get("tenant").and_then(|v| v.as_str()).unwrap_or(""); - - if agent_id.is_empty() || action_name.is_empty() || resource_type.is_empty() { - return Err(format!( - "GenerateCedarPolicy: missing required fields for entity '{entity_id}'" - )); - } - - // Parse scope_matrix from params, or build a default matrix based on the legacy scope string. - let matrix: temper_authz::PolicyScopeMatrix = - if let Some(matrix_val) = params.get("scope_matrix") { - serde_json::from_value(matrix_val.clone()).map_err(|e| { - format!("GenerateCedarPolicy: invalid scope_matrix for entity '{entity_id}': {e}") - })? - } else { - match scope { - "narrow" => temper_authz::PolicyScopeMatrix { - principal: temper_authz::PrincipalScope::ThisAgent, - action: temper_authz::ActionScope::ThisAction, - resource: temper_authz::ResourceScope::ThisResource, - duration: temper_authz::DurationScope::Always, - agent_type_value: None, - role_value: None, - session_id: None, - }, - "broad" => temper_authz::PolicyScopeMatrix { - principal: temper_authz::PrincipalScope::ThisAgent, - action: temper_authz::ActionScope::AllActionsOnType, - resource: temper_authz::ResourceScope::AnyOfType, - duration: temper_authz::DurationScope::Always, - agent_type_value: None, - role_value: None, - session_id: None, - }, - _ => temper_authz::PolicyScopeMatrix::default_for(None), - } - }; - temper_authz::validate_policy_scope_matrix(&matrix).map_err(|e| { - format!("GenerateCedarPolicy: invalid scope_matrix for entity '{entity_id}': {e}") - })?; - let principal_kind = params - .get("principal_kind") - .and_then(|v| v.as_str()) - .unwrap_or("Agent"); - let generated_policy = temper_authz::generate_cedar_from_matrix( - agent_id, - principal_kind, - action_name, - resource_type, - resource_id, - &matrix, - ); - - tracing::info!( - entity_id = entity_id, - tenant = tenant, - scope = scope, - "GenerateCedarPolicy hook: generated policy, validating and loading" - ); - - // Validate and reload the per-tenant policy set. - { - let Ok(mut policies) = state.server.tenant_policies.write() else { - return Err("tenant_policies lock poisoned".to_string()); - }; - let entry = policies.entry(tenant.to_string()).or_default(); - if !entry.is_empty() { - entry.push('\n'); - } - entry.push_str(&generated_policy); - - let tenant_text = entry.clone(); - if let Err(e) = state - .server - .authz - .reload_tenant_policies(tenant, &tenant_text) - { - tracing::error!(error = %e, "GenerateCedarPolicy: failed to reload policies"); - return Err(format!("Failed to reload policies: {e}")); - } - } - - tracing::info!( - entity_id = entity_id, - "GenerateCedarPolicy hook: policy loaded successfully" - ); - Ok(()) + generate_cedar::handle_generate_cedar_from_fields(entity_id, params, &state.server).await } // --------------------------------------------------------------------------- @@ -266,8 +155,9 @@ pub struct PlatformEffectHandler { pub spec_store: Arc>, } +#[async_trait::async_trait] impl CustomEffectHandler for PlatformEffectHandler { - fn handle( + async fn handle( &self, effect_name: &str, entity_type: &str, @@ -278,9 +168,11 @@ impl CustomEffectHandler for PlatformEffectHandler { match effect_name { "GenerateCedarPolicy" => { generate_cedar::handle_generate_cedar_from_fields(entity_id, entity_fields, server) + .await } "DispatchCallback" => { - governance_callback::handle_dispatch_callback(entity_fields, server) + governance_callback::handle_dispatch_callback(entity_id, entity_fields, server) + .await } _ => { tracing::debug!( @@ -336,8 +228,8 @@ fn generate_cedar_permit( mod tests { use super::*; - #[test] - fn test_dispatch_unknown_effect_is_ok() { + #[tokio::test] + async fn test_dispatch_unknown_effect_is_ok() { let state = PlatformState::new(None); let result = dispatch_custom_effect( "UnknownEffect", @@ -345,7 +237,8 @@ mod tests { "t-1", &serde_json::json!({}), &state, - ); + ) + .await; assert!(result.is_ok()); } @@ -376,8 +269,8 @@ mod tests { assert!(!policy.contains("submitOrder")); } - #[test] - fn test_dispatch_generate_cedar_policy_missing_fields() { + #[tokio::test] + async fn test_dispatch_generate_cedar_policy_missing_fields() { let state = PlatformState::new(None); let result = dispatch_custom_effect( "GenerateCedarPolicy", @@ -385,13 +278,78 @@ mod tests { "gd-1", &serde_json::json!({}), &state, - ); + ) + .await; assert!(result.is_err()); assert!(result.unwrap_err().contains("missing required fields")); } - #[test] - fn test_dispatch_deploy_specs_no_store_entry() { + #[tokio::test] + async fn test_generate_cedar_policy_skips_api_owned_publication() { + let state = PlatformState::new(None); + let result = dispatch_custom_effect( + "GenerateCedarPolicy", + "GovernanceDecision", + "gd-api-owned", + &serde_json::json!({"policy_already_published": true}), + &state, + ) + .await; + assert!( + result.is_ok(), + "API-owned policy must not be generated twice" + ); + } + + #[tokio::test] + async fn test_generate_cedar_policy_persists_one_stable_decision_entry() { + let mut state = PlatformState::new(None); + let mut path = std::env::temp_dir(); + path.push(format!( + "temper-generated-policy-{}.db", + uuid::Uuid::new_v4() + )); + let store = + temper_store_turso::TursoEventStore::new(&format!("file:{}", path.display()), None) + .await + .expect("create durable test store"); + state + .server + .set_storage_stack(temper_server::storage::StorageStack::from_turso( + store.clone(), + )); + let fields = serde_json::json!({ + "agent_id": "agent-1", + "action_name": "read", + "resource_type": "Document", + "resource_id": "doc-1", + "scope": "narrow", + "tenant": "tenant-policy-test", + "decided_by": "reviewer-1", + }); + + for _ in 0..2 { + dispatch_custom_effect( + "GenerateCedarPolicy", + "GovernanceDecision", + "gd-stable", + &fields, + &state, + ) + .await + .expect("publish generated policy generation"); + } + + let policies = store + .load_policies_for_tenant("tenant-policy-test") + .await + .expect("read durable policies"); + assert_eq!(policies.len(), 1); + assert_eq!(policies[0].policy_id, "decision:gd-stable"); + } + + #[tokio::test] + async fn test_dispatch_deploy_specs_no_store_entry() { let state = PlatformState::new(None); let result = dispatch_custom_effect( "DeploySpecs", @@ -399,7 +357,8 @@ mod tests { "t-1", &serde_json::json!({}), &state, - ); + ) + .await; // No specs in store → error assert!(result.is_err()); assert!(result.unwrap_err().contains("no specs found")); diff --git a/crates/temper-platform/src/hooks/generate_cedar.rs b/crates/temper-platform/src/hooks/generate_cedar.rs index f87291bb7..63c0baac8 100644 --- a/crates/temper-platform/src/hooks/generate_cedar.rs +++ b/crates/temper-platform/src/hooks/generate_cedar.rs @@ -4,11 +4,22 @@ use temper_server::ServerState; /// /// Reads agent_id, action_name, resource_type, resource_id, scope, tenant, /// and scope_matrix from the GovernanceDecision entity's merged fields. -pub(super) fn handle_generate_cedar_from_fields( +pub(super) async fn handle_generate_cedar_from_fields( entity_id: &str, fields: &serde_json::Value, server: &ServerState, ) -> Result<(), String> { + if fields + .get("policy_already_published") + .and_then(serde_json::Value::as_bool) + .unwrap_or(false) + { + tracing::debug!( + entity_id, + "GenerateCedarPolicy hook skipped because the API already published this policy" + ); + return Ok(()); + } let agent_id = fields .get("agent_id") .and_then(|v| v.as_str()) @@ -88,22 +99,18 @@ pub(super) fn handle_generate_cedar_from_fields( "GenerateCedarPolicy hook: generated policy, validating and loading" ); - { - let Ok(mut policies) = server.tenant_policies.write() else { - return Err("tenant_policies lock poisoned".to_string()); - }; - let entry = policies.entry(tenant.to_string()).or_default(); - if !entry.is_empty() { - entry.push('\n'); - } - entry.push_str(&generated_policy); - - let tenant_text = entry.clone(); - if let Err(e) = server.authz.reload_tenant_policies(tenant, &tenant_text) { - tracing::error!(error = %e, "GenerateCedarPolicy: failed to reload policies"); - return Err(format!("Failed to reload policies: {e}")); - } - } + let created_by = fields + .get("decided_by") + .and_then(|value| value.as_str()) + .unwrap_or("governance-decision"); + temper_server::authz::publish_policy_entry_generation( + server, + tenant, + &format!("decision:{entity_id}"), + &generated_policy, + created_by, + ) + .await?; tracing::info!( entity_id, diff --git a/crates/temper-platform/src/hooks/governance_callback.rs b/crates/temper-platform/src/hooks/governance_callback.rs index 33999f18b..41b05a4e1 100644 --- a/crates/temper-platform/src/hooks/governance_callback.rs +++ b/crates/temper-platform/src/hooks/governance_callback.rs @@ -1,3 +1,4 @@ +use sha2::{Digest, Sha256}; use temper_runtime::TenantId; use temper_server::ServerState; use temper_server::request_context::AgentContext; @@ -12,7 +13,8 @@ use temper_server::request_context::AgentContext; /// now triggers `DispatchCallback`, so replaying callback wiring after a /// decision has already been approved or denied will immediately redeliver /// the resolution to the waiting target entity. -pub(super) fn handle_dispatch_callback( +pub(super) async fn handle_dispatch_callback( + source_entity_id: &str, entity_fields: &serde_json::Value, server: &ServerState, ) -> Result<(), String> { @@ -83,51 +85,94 @@ pub(super) fn handle_dispatch_callback( "DispatchCallback: dispatching callback" ); - let server = server.clone(); - let tenant = callback_tenant.to_string(); - let entity_set = callback_entity_set.to_string(); - let entity_id = callback_entity_id.to_string(); - let action = callback_action.to_string(); - - tokio::spawn(async move { - // determinism-ok: async callback dispatch for governance decision resolution - let tid = TenantId::new(&tenant); - let entity_type = resolve_callback_entity_type(&server, &tid, &entity_set); - if let Err(e) = server - .dispatch_tenant_action( - &tid, - &entity_type, - &entity_id, - &action, - params, - &AgentContext::for_service("governance-service"), - ) - .await - { + let tid = TenantId::new(callback_tenant); + let entity_type = resolve_callback_entity_type(server, &tid, callback_entity_set); + let mut agent_context = AgentContext::for_service("governance-service"); + agent_context.idempotency_key = Some(callback_idempotency_key( + source_entity_id, + status, + callback_tenant, + &entity_type, + callback_entity_id, + callback_action, + )); + let response = server + .dispatch_tenant_action( + &tid, + &entity_type, + callback_entity_id, + callback_action, + params, + &agent_context, + ) + .await + .map_err(|error| { tracing::error!( - error = %e, - tenant, - entity_set, + error = %error, + tenant = callback_tenant, + entity_set = callback_entity_set, entity_type, - entity_id, - action, + entity_id = callback_entity_id, + action = callback_action, "DispatchCallback: failed to dispatch callback action" ); - } else { - tracing::info!( - tenant, - entity_set, - entity_type, - entity_id, - action, - "DispatchCallback: callback action dispatched successfully" - ); - } - }); + format!("DispatchCallback: failed to dispatch callback action: {error}") + })?; + if !response.success { + let error = response + .error + .as_deref() + .unwrap_or("callback action returned success=false without an error"); + tracing::error!( + error, + tenant = callback_tenant, + entity_set = callback_entity_set, + entity_type, + entity_id = callback_entity_id, + action = callback_action, + "DispatchCallback: callback action was rejected" + ); + return Err(format!( + "DispatchCallback: callback action was rejected: {error}" + )); + } + + tracing::info!( + tenant = callback_tenant, + entity_set = callback_entity_set, + entity_type, + entity_id = callback_entity_id, + action = callback_action, + "DispatchCallback: callback action dispatched successfully" + ); Ok(()) } +fn callback_idempotency_key( + source_entity_id: &str, + status: &str, + callback_tenant: &str, + callback_entity_type: &str, + callback_entity_id: &str, + callback_action: &str, +) -> String { + let mut hasher = Sha256::new(); + for part in [ + "DispatchCallback", + source_entity_id, + status, + callback_tenant, + callback_entity_type, + callback_entity_id, + callback_action, + ] { + hasher.update((part.len() as u64).to_be_bytes()); + hasher.update(part.as_bytes()); + } + format!("governance-callback:{:x}", hasher.finalize()) +} + /// Resolve a callback target from an OData entity-set name to a governed entity type. /// /// Callbacks are registered over HTTP and therefore store OData-facing entity set names @@ -164,272 +209,5 @@ fn resolve_callback_entity_type( } #[cfg(test)] -mod tests { - use std::collections::BTreeMap; - use std::time::Duration; - - use super::*; - use crate::bootstrap::{SYSTEM_TENANT, bootstrap_system_tenant}; - use crate::state::PlatformState; - use temper_spec::csdl::{CsdlDocument, parse_csdl}; - - fn session_callback_csdl() -> (CsdlDocument, String) { - let xml = r#" - - - - - - - - - - - - - "#; - (parse_csdl(xml).unwrap(), xml.to_string()) - } - - const SESSION_IOA: &str = r#" -[automaton] -name = "Session" -initial = "WaitingForApproval" -states = ["WaitingForApproval", "Executing", "Failed"] - -[[action]] -name = "ResumeAfterApproval" -from = ["WaitingForApproval"] -to = "Executing" -kind = "input" - -[[action]] -name = "Fail" -from = ["WaitingForApproval"] -to = "Failed" -kind = "input" -params = ["error_message"] -"#; - - #[test] - fn callback_entity_set_resolution_uses_registry_mapping() { - let state = PlatformState::new(None); - let (csdl, xml) = session_callback_csdl(); - state.registry.write().unwrap().register_tenant( - "default", - csdl, - xml, - &[("Session", SESSION_IOA)], - ); - - let tenant = TenantId::new("default"); - assert_eq!( - resolve_callback_entity_type(&state.server, &tenant, "Sessions"), - "Session" - ); - } - - #[test] - fn callback_entity_set_resolution_preserves_entity_type_inputs() { - let state = PlatformState::new(None); - let (csdl, xml) = session_callback_csdl(); - state.registry.write().unwrap().register_tenant( - "default", - csdl, - xml, - &[("Session", SESSION_IOA)], - ); - - let tenant = TenantId::new("default"); - assert_eq!( - resolve_callback_entity_type(&state.server, &tenant, "Session"), - "Session" - ); - } - - #[tokio::test] - async fn approved_governance_decision_dispatches_callback_registered_with_entity_set_name() { - let state = PlatformState::new(None); - bootstrap_system_tenant(&state, &BTreeMap::new()); - - let (csdl, xml) = session_callback_csdl(); - state.registry.write().unwrap().register_tenant( - "default", - csdl, - xml, - &[("Session", SESSION_IOA)], - ); - - let system_tenant = TenantId::new(SYSTEM_TENANT); - let app_tenant = TenantId::new("default"); - let decision_id = "gd-callback-test"; - let session_id = "ss-callback-test"; - - state - .server - .dispatch_tenant_action( - &system_tenant, - "GovernanceDecision", - decision_id, - "CreateGovernanceDecision", - serde_json::json!({ - "tenant": "default", - "agent_id": "bot-openpaw", - "action_name": "temper.submit_specs", - "resource_type": "Session", - "resource_id": session_id, - "denial_reason": "", - "scope": "narrow", - "pending_decision_id": decision_id, - }), - &AgentContext::for_service("governance-service"), - ) - .await - .expect("governance decision should be created"); - - state - .server - .dispatch_tenant_action( - &system_tenant, - "GovernanceDecision", - decision_id, - "RegisterCallback", - serde_json::json!({ - "callback_tenant": "default", - "callback_entity_set": "Sessions", - "callback_entity_id": session_id, - "callback_on_approve": "ResumeAfterApproval", - "callback_on_deny": "Fail", - }), - &AgentContext::for_service("governance-service"), - ) - .await - .expect("callback should register"); - - state - .server - .dispatch_tenant_action( - &system_tenant, - "GovernanceDecision", - decision_id, - "Approve", - serde_json::json!({ - "decided_by": "human-reviewer", - "scope": "narrow", - "generated_policy": "", - }), - &AgentContext::for_service("governance-service"), - ) - .await - .expect("approval should succeed"); - - tokio::time::timeout(Duration::from_secs(2), async { - loop { - if let Ok(entity) = state - .server - .get_tenant_entity_state(&app_tenant, "Session", session_id) - .await - && entity.state.status == "Executing" - { - break; - } - tokio::time::sleep(Duration::from_millis(25)).await; - } - }) - .await - .expect("callback should resume the target session"); - } - - #[tokio::test] - async fn late_callback_registration_replays_approved_governance_decision() { - let state = PlatformState::new(None); - bootstrap_system_tenant(&state, &BTreeMap::new()); - - let (csdl, xml) = session_callback_csdl(); - state.registry.write().unwrap().register_tenant( - "default", - csdl, - xml, - &[("Session", SESSION_IOA)], - ); - - let system_tenant = TenantId::new(SYSTEM_TENANT); - let app_tenant = TenantId::new("default"); - let decision_id = "gd-late-callback-test"; - let session_id = "ss-late-callback-test"; - - state - .server - .dispatch_tenant_action( - &system_tenant, - "GovernanceDecision", - decision_id, - "CreateGovernanceDecision", - serde_json::json!({ - "tenant": "default", - "agent_id": "bot-openpaw", - "action_name": "temper.submit_specs", - "resource_type": "Session", - "resource_id": session_id, - "denial_reason": "", - "scope": "narrow", - "pending_decision_id": decision_id, - }), - &AgentContext::for_service("governance-service"), - ) - .await - .expect("governance decision should be created"); - - state - .server - .dispatch_tenant_action( - &system_tenant, - "GovernanceDecision", - decision_id, - "Approve", - serde_json::json!({ - "decided_by": "human-reviewer", - "scope": "narrow", - "generated_policy": "", - }), - &AgentContext::for_service("governance-service"), - ) - .await - .expect("approval should succeed before callback registration"); - - state - .server - .dispatch_tenant_action( - &system_tenant, - "GovernanceDecision", - decision_id, - "RegisterCallback", - serde_json::json!({ - "callback_tenant": "default", - "callback_entity_set": "Sessions", - "callback_entity_id": session_id, - "callback_on_approve": "ResumeAfterApproval", - "callback_on_deny": "Fail", - }), - &AgentContext::for_service("governance-service"), - ) - .await - .expect("late callback registration should replay the approval callback"); - - tokio::time::timeout(Duration::from_secs(2), async { - loop { - if let Ok(entity) = state - .server - .get_tenant_entity_state(&app_tenant, "Session", session_id) - .await - && entity.state.status == "Executing" - { - break; - } - tokio::time::sleep(Duration::from_millis(25)).await; - } - }) - .await - .expect("late callback registration should resume the target session"); - } -} +#[path = "governance_callback_test.rs"] +mod tests; diff --git a/crates/temper-platform/src/hooks/governance_callback_test.rs b/crates/temper-platform/src/hooks/governance_callback_test.rs new file mode 100644 index 000000000..0bf6120ff --- /dev/null +++ b/crates/temper-platform/src/hooks/governance_callback_test.rs @@ -0,0 +1,385 @@ +use std::collections::BTreeMap; +use std::time::Duration; + +use super::*; +use crate::bootstrap::{SYSTEM_TENANT, bootstrap_system_tenant}; +use crate::state::PlatformState; +use temper_spec::csdl::{CsdlDocument, parse_csdl}; + +fn session_callback_csdl() -> (CsdlDocument, String) { + let xml = r#" + + + + + + + + + + + + + "#; + (parse_csdl(xml).unwrap(), xml.to_string()) +} + +const SESSION_IOA: &str = r#" +[automaton] +name = "Session" +initial = "WaitingForApproval" +states = ["WaitingForApproval", "Executing", "Failed"] + +[[action]] +name = "ResumeAfterApproval" +from = ["WaitingForApproval"] +to = "Executing" +kind = "input" + +[[action]] +name = "Fail" +from = ["WaitingForApproval"] +to = "Failed" +kind = "input" +params = ["error_message"] +"#; + +const RETRY_CALLBACK_IOA: &str = r#" +[automaton] +name = "Session" +initial = "Executing" +states = ["Executing"] + +[[state]] +name = "callback_count" +type = "counter" +initial = "0" + +[[action]] +name = "ResumeAfterApproval" +from = ["Executing"] +to = "Executing" +kind = "input" +effect = [{ type = "increment", var = "callback_count" }] +"#; + +#[test] +fn callback_entity_set_resolution_uses_registry_mapping() { + let state = PlatformState::new(None); + let (csdl, xml) = session_callback_csdl(); + state.registry.write().unwrap().register_tenant( + "default", + csdl, + xml, + &[("Session", SESSION_IOA)], + ); + + let tenant = TenantId::new("default"); + assert_eq!( + resolve_callback_entity_type(&state.server, &tenant, "Sessions"), + "Session" + ); +} + +#[test] +fn callback_entity_set_resolution_preserves_entity_type_inputs() { + let state = PlatformState::new(None); + let (csdl, xml) = session_callback_csdl(); + state.registry.write().unwrap().register_tenant( + "default", + csdl, + xml, + &[("Session", SESSION_IOA)], + ); + + let tenant = TenantId::new("default"); + assert_eq!( + resolve_callback_entity_type(&state.server, &tenant, "Session"), + "Session" + ); +} + +#[tokio::test] +async fn callback_dispatch_failure_is_returned_before_effect_acknowledgement() { + let state = PlatformState::new(None); + let result = handle_dispatch_callback( + "gd-missing-callback-test", + &serde_json::json!({ + "Status": "Approved", + "callback_tenant": "default", + "callback_entity_set": "MissingEntities", + "callback_entity_id": "missing-callback-target", + "callback_on_approve": "ResumeAfterApproval", + }), + &state.server, + ) + .await; + + assert!( + result.is_err(), + "a failed callback must keep the durable custom-effect receipt retryable" + ); +} + +#[tokio::test] +async fn rejected_callback_response_is_returned_before_effect_acknowledgement() { + let state = PlatformState::new(None); + let (csdl, xml) = session_callback_csdl(); + state.registry.write().unwrap().register_tenant( + "default", + csdl, + xml, + &[("Session", SESSION_IOA)], + ); + + let result = handle_dispatch_callback( + "gd-rejected-callback-test", + &serde_json::json!({ + "Status": "Approved", + "callback_tenant": "default", + "callback_entity_set": "Sessions", + "callback_entity_id": "rejected-callback-target", + "callback_on_approve": "MissingAction", + }), + &state.server, + ) + .await; + + assert!( + result.is_err(), + "a rejected governed callback must keep the durable custom-effect receipt retryable" + ); +} + +#[tokio::test] +async fn retried_callback_uses_one_stable_target_idempotency_key() { + let state = PlatformState::new(None); + let (csdl, xml) = session_callback_csdl(); + state.registry.write().unwrap().register_tenant( + "default", + csdl, + xml, + &[("Session", RETRY_CALLBACK_IOA)], + ); + let fields = serde_json::json!({ + "Status": "Approved", + "callback_tenant": "default", + "callback_entity_set": "Sessions", + "callback_entity_id": "retried-callback-target", + "callback_on_approve": "ResumeAfterApproval", + }); + + handle_dispatch_callback("gd-retried-callback-test", &fields, &state.server) + .await + .expect("first callback dispatch should succeed"); + handle_dispatch_callback("gd-retried-callback-test", &fields, &state.server) + .await + .expect("replayed callback dispatch should be idempotent"); + + let entity = state + .server + .get_tenant_entity_state( + &TenantId::new("default"), + "Session", + "retried-callback-target", + ) + .await + .expect("callback target should exist"); + assert_eq!( + entity.state.counters.get("callback_count"), + Some(&1), + "a source-effect retry must not apply the callback twice" + ); +} + +#[tokio::test] +async fn approved_governance_decision_dispatches_callback_registered_with_entity_set_name() { + let state = PlatformState::new(None); + bootstrap_system_tenant(&state, &BTreeMap::new()); + + let (csdl, xml) = session_callback_csdl(); + state.registry.write().unwrap().register_tenant( + "default", + csdl, + xml, + &[("Session", SESSION_IOA)], + ); + + let system_tenant = TenantId::new(SYSTEM_TENANT); + let app_tenant = TenantId::new("default"); + let decision_id = "gd-callback-test"; + let session_id = "ss-callback-test"; + + state + .server + .dispatch_tenant_action( + &system_tenant, + "GovernanceDecision", + decision_id, + "CreateGovernanceDecision", + serde_json::json!({ + "tenant": "default", + "agent_id": "bot-openpaw", + "action_name": "temper.submit_specs", + "resource_type": "Session", + "resource_id": session_id, + "denial_reason": "", + "scope": "narrow", + "pending_decision_id": decision_id, + }), + &AgentContext::for_service("governance-service"), + ) + .await + .expect("governance decision should be created"); + + state + .server + .dispatch_tenant_action( + &system_tenant, + "GovernanceDecision", + decision_id, + "RegisterCallback", + serde_json::json!({ + "callback_tenant": "default", + "callback_entity_set": "Sessions", + "callback_entity_id": session_id, + "callback_on_approve": "ResumeAfterApproval", + "callback_on_deny": "Fail", + }), + &AgentContext::for_service("governance-service"), + ) + .await + .expect("callback should register"); + + let approval = state + .server + .dispatch_tenant_action( + &system_tenant, + "GovernanceDecision", + decision_id, + "Approve", + serde_json::json!({ + "decided_by": "human-reviewer", + "scope": "narrow", + "generated_policy": "", + "policy_already_published": true, + }), + &AgentContext::for_service("governance-service"), + ) + .await + .expect("approval should succeed"); + assert!( + approval.success, + "approval post-effects should succeed: {:?}", + approval.error + ); + + tokio::time::timeout(Duration::from_secs(2), async { + loop { + if let Ok(entity) = state + .server + .get_tenant_entity_state(&app_tenant, "Session", session_id) + .await + && entity.state.status == "Executing" + { + break; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + }) + .await + .expect("callback should resume the target session"); +} + +#[tokio::test] +async fn late_callback_registration_replays_approved_governance_decision() { + let state = PlatformState::new(None); + bootstrap_system_tenant(&state, &BTreeMap::new()); + + let (csdl, xml) = session_callback_csdl(); + state.registry.write().unwrap().register_tenant( + "default", + csdl, + xml, + &[("Session", SESSION_IOA)], + ); + + let system_tenant = TenantId::new(SYSTEM_TENANT); + let app_tenant = TenantId::new("default"); + let decision_id = "gd-late-callback-test"; + let session_id = "ss-late-callback-test"; + + state + .server + .dispatch_tenant_action( + &system_tenant, + "GovernanceDecision", + decision_id, + "CreateGovernanceDecision", + serde_json::json!({ + "tenant": "default", + "agent_id": "bot-openpaw", + "action_name": "temper.submit_specs", + "resource_type": "Session", + "resource_id": session_id, + "denial_reason": "", + "scope": "narrow", + "pending_decision_id": decision_id, + }), + &AgentContext::for_service("governance-service"), + ) + .await + .expect("governance decision should be created"); + + state + .server + .dispatch_tenant_action( + &system_tenant, + "GovernanceDecision", + decision_id, + "Approve", + serde_json::json!({ + "decided_by": "human-reviewer", + "scope": "narrow", + "generated_policy": "", + }), + &AgentContext::for_service("governance-service"), + ) + .await + .expect("approval should succeed before callback registration"); + + state + .server + .dispatch_tenant_action( + &system_tenant, + "GovernanceDecision", + decision_id, + "RegisterCallback", + serde_json::json!({ + "callback_tenant": "default", + "callback_entity_set": "Sessions", + "callback_entity_id": session_id, + "callback_on_approve": "ResumeAfterApproval", + "callback_on_deny": "Fail", + }), + &AgentContext::for_service("governance-service"), + ) + .await + .expect("late callback registration should replay the approval callback"); + + tokio::time::timeout(Duration::from_secs(2), async { + loop { + if let Ok(entity) = state + .server + .get_tenant_entity_state(&app_tenant, "Session", session_id) + .await + && entity.state.status == "Executing" + { + break; + } + tokio::time::sleep(Duration::from_millis(25)).await; + } + }) + .await + .expect("late callback registration should resume the target session"); +} diff --git a/crates/temper-platform/src/os_apps/agent_bootstrap.rs b/crates/temper-platform/src/os_apps/agent_bootstrap.rs index ee03330b9..364f4a867 100644 --- a/crates/temper-platform/src/os_apps/agent_bootstrap.rs +++ b/crates/temper-platform/src/os_apps/agent_bootstrap.rs @@ -7,12 +7,20 @@ use crate::state::PlatformState; use super::{AgentDefinition, content_sha256, state_field_str}; +mod soul_refresh; + +#[cfg(test)] +pub(super) use soul_refresh::decide_agent_soul_refresh; +pub(super) use soul_refresh::{AgentSoulRefreshDecision, bootstrapped_agent_soul_entity_id}; +use soul_refresh::{inspect_agent_soul_refresh, slugify_bootstrapped_agent_name}; + pub(super) async fn bootstrap_agents( state: &PlatformState, tenant_id: &TenantId, tenant: &str, agents: &[AgentDefinition], app_id: Option<&str>, + agent_ctx: &temper_server::request_context::AgentContext, ) -> (Vec, BTreeMap) { let mut bootstrapped = Vec::new(); let mut agent_uuid_map: BTreeMap = BTreeMap::new(); @@ -51,14 +59,12 @@ pub(super) async fn bootstrap_agents( registry.get_spec(tenant_id, "Agent").is_some() }; - let agent_ctx = temper_server::request_context::AgentContext::for_service("platform-bootstrap"); - for agent in agents { let stable_soul_id = bootstrapped_agent_soul_entity_id(&agent.name); if let Ok(resp) = state .server - .get_tenant_entity_state(tenant_id, "Soul", &stable_soul_id) + .get_tenant_entity_state_in_generation(tenant_id, "Soul", &stable_soul_id, agent_ctx) .await { if let Some(file_id) = resp @@ -69,7 +75,15 @@ pub(super) async fn bootstrap_agents( .and_then(|v| v.as_str()) { let desired_hash = content_sha256(agent.content.as_bytes()); - match inspect_agent_soul_refresh(state, tenant_id, file_id, &desired_hash).await { + match inspect_agent_soul_refresh( + state, + tenant_id, + file_id, + &desired_hash, + agent_ctx, + ) + .await + { Ok(AgentSoulRefreshDecision::AlreadyCurrent) => { tracing::debug!( tenant, @@ -92,7 +106,7 @@ pub(super) async fn bootstrap_agents( if let Err(error) = ensure_inline_file_uploaded( state, tenant_id, - &agent_ctx, + agent_ctx, file_id, &format!("{}.soul.md", slugify_bootstrapped_agent_name(&agent.name)), agent.content.as_bytes(), @@ -125,7 +139,7 @@ pub(super) async fn bootstrap_agents( agent_uuid_map.insert(agent.name.clone(), stable_soul_id.clone()); bootstrapped.push(agent.name.clone()); if has_agents && let Some(app_id) = app_id { - set_agent_source_app(state, tenant_id, tenant, &agent_ctx, &agent.name, app_id) + set_agent_source_app(state, tenant_id, tenant, agent_ctx, &agent.name, app_id) .await; } continue; @@ -136,7 +150,7 @@ pub(super) async fn bootstrap_agents( for id in &existing_ids { if let Ok(resp) = state .server - .get_tenant_entity_state(tenant_id, "Soul", id) + .get_tenant_entity_state_in_generation(tenant_id, "Soul", id, agent_ctx) .await && let Some(name) = state_field_str(&resp.state.fields, &["Name", "name"]) && name.eq_ignore_ascii_case(&agent.name) @@ -149,7 +163,14 @@ pub(super) async fn bootstrap_agents( .and_then(|v| v.as_str()) { let desired_hash = content_sha256(agent.content.as_bytes()); - match inspect_agent_soul_refresh(state, tenant_id, file_id, &desired_hash).await + match inspect_agent_soul_refresh( + state, + tenant_id, + file_id, + &desired_hash, + agent_ctx, + ) + .await { Ok(AgentSoulRefreshDecision::AlreadyCurrent) => { tracing::debug!( @@ -171,7 +192,7 @@ pub(super) async fn bootstrap_agents( if let Err(error) = ensure_inline_file_uploaded( state, tenant_id, - &agent_ctx, + agent_ctx, file_id, &format!("{}.soul.md", agent.name.to_lowercase().replace(' ', "-")), agent.content.as_bytes(), @@ -209,7 +230,7 @@ pub(super) async fn bootstrap_agents( bootstrapped.push(agent.name.clone()); if has_agents && let Some(app_id) = app_id { - set_agent_source_app(state, tenant_id, tenant, &agent_ctx, &agent.name, app_id) + set_agent_source_app(state, tenant_id, tenant, agent_ctx, &agent.name, app_id) .await; } continue; @@ -223,7 +244,7 @@ pub(super) async fn bootstrap_agents( if let Err(error) = ensure_inline_file_uploaded( state, tenant_id, - &agent_ctx, + agent_ctx, &temp_file_id, &file_name, agent.content.as_bytes(), @@ -242,7 +263,13 @@ pub(super) async fn bootstrap_agents( let new_soul_id = stable_soul_id; match state .server - .get_or_create_tenant_entity(tenant_id, "Soul", &new_soul_id, json!({})) + .get_or_create_tenant_entity_in_generation( + tenant_id, + "Soul", + &new_soul_id, + json!({}), + agent_ctx, + ) .await { Ok(_) => { @@ -259,7 +286,7 @@ pub(super) async fn bootstrap_agents( "description": agent.description, "content_file_id": temp_file_id, }), - agent_ctx: &agent_ctx, + agent_ctx, await_integration: false, await_reactions: true, }) @@ -281,7 +308,7 @@ pub(super) async fn bootstrap_agents( entity_id: &soul_id, action: "Publish", params: json!({}), - agent_ctx: &agent_ctx, + agent_ctx, await_integration: false, await_reactions: true, }) @@ -292,7 +319,7 @@ pub(super) async fn bootstrap_agents( bootstrapped.push(agent.name.clone()); if has_agents && let Some(app_id) = app_id { - set_agent_source_app(state, tenant_id, tenant, &agent_ctx, &agent.name, app_id) + set_agent_source_app(state, tenant_id, tenant, agent_ctx, &agent.name, app_id) .await; } } @@ -322,7 +349,7 @@ async fn set_agent_source_app( for id in &agent_ids { if let Ok(resp) = state .server - .get_tenant_entity_state(tenant_id, "Agent", id) + .get_tenant_entity_state_in_generation(tenant_id, "Agent", id, agent_ctx) .await && let Some(name) = state_field_str(&resp.state.fields, &["Name", "name"]) && name.eq_ignore_ascii_case(agent_name) @@ -378,7 +405,7 @@ async fn ensure_inline_file_uploaded( ) -> Result<(), String> { if !state .server - .ensure_entity_loaded(tenant_id, "File", file_id) + .ensure_entity_loaded_in_generation(tenant_id, "File", file_id, agent_ctx) .await { state @@ -398,7 +425,7 @@ async fn ensure_inline_file_uploaded( let response = state .server - .get_tenant_entity_state(tenant_id, "File", file_id) + .get_tenant_entity_state_in_generation(tenant_id, "File", file_id, agent_ctx) .await .map_err(|e| format!("failed to load File('{file_id}') actor: {e}"))?; @@ -433,67 +460,3 @@ async fn ensure_inline_file_uploaded( Ok(()) } - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(super) enum AgentSoulRefreshDecision { - Upload, - AlreadyCurrent, - PreserveCustomized, -} - -fn slugify_bootstrapped_agent_name(name: &str) -> String { - name.trim().to_lowercase().replace(' ', "-") -} - -pub(super) fn bootstrapped_agent_soul_entity_id(name: &str) -> String { - format!( - "sl-bootstrap-agent-soul-{}", - slugify_bootstrapped_agent_name(name) - ) -} - -async fn inspect_agent_soul_refresh( - state: &PlatformState, - tenant_id: &TenantId, - file_id: &str, - desired_hash: &str, -) -> Result { - let response = state - .server - .get_tenant_entity_state(tenant_id, "File", file_id) - .await - .map_err(|e| format!("failed to inspect File('{file_id}'): {e}"))?; - - let has_content = response - .state - .booleans - .get("has_content") - .copied() - .unwrap_or(false); - let current_hash = response - .state - .fields - .get("content_hash") - .and_then(|value| value.as_str()) - .unwrap_or_default(); - - Ok(decide_agent_soul_refresh( - has_content, - current_hash, - desired_hash, - )) -} - -pub(super) fn decide_agent_soul_refresh( - has_content: bool, - current_hash: &str, - desired_hash: &str, -) -> AgentSoulRefreshDecision { - if !has_content || current_hash.is_empty() { - AgentSoulRefreshDecision::Upload - } else if current_hash == desired_hash { - AgentSoulRefreshDecision::AlreadyCurrent - } else { - AgentSoulRefreshDecision::PreserveCustomized - } -} diff --git a/crates/temper-platform/src/os_apps/agent_bootstrap/soul_refresh.rs b/crates/temper-platform/src/os_apps/agent_bootstrap/soul_refresh.rs new file mode 100644 index 000000000..2c630dc6d --- /dev/null +++ b/crates/temper-platform/src/os_apps/agent_bootstrap/soul_refresh.rs @@ -0,0 +1,68 @@ +use temper_runtime::tenant::TenantId; + +use crate::state::PlatformState; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(in crate::os_apps) enum AgentSoulRefreshDecision { + Upload, + AlreadyCurrent, + PreserveCustomized, +} + +pub(super) fn slugify_bootstrapped_agent_name(name: &str) -> String { + name.trim().to_lowercase().replace(' ', "-") +} + +pub(in crate::os_apps) fn bootstrapped_agent_soul_entity_id(name: &str) -> String { + format!( + "sl-bootstrap-agent-soul-{}", + slugify_bootstrapped_agent_name(name) + ) +} + +pub(super) async fn inspect_agent_soul_refresh( + state: &PlatformState, + tenant_id: &TenantId, + file_id: &str, + desired_hash: &str, + agent_ctx: &temper_server::request_context::AgentContext, +) -> Result { + let response = state + .server + .get_tenant_entity_state_in_generation(tenant_id, "File", file_id, agent_ctx) + .await + .map_err(|e| format!("failed to inspect File('{file_id}'): {e}"))?; + + let has_content = response + .state + .booleans + .get("has_content") + .copied() + .unwrap_or(false); + let current_hash = response + .state + .fields + .get("content_hash") + .and_then(|value| value.as_str()) + .unwrap_or_default(); + + Ok(decide_agent_soul_refresh( + has_content, + current_hash, + desired_hash, + )) +} + +pub(in crate::os_apps) fn decide_agent_soul_refresh( + has_content: bool, + current_hash: &str, + desired_hash: &str, +) -> AgentSoulRefreshDecision { + if !has_content || current_hash.is_empty() { + AgentSoulRefreshDecision::Upload + } else if current_hash == desired_hash { + AgentSoulRefreshDecision::AlreadyCurrent + } else { + AgentSoulRefreshDecision::PreserveCustomized + } +} diff --git a/crates/temper-platform/src/os_apps/discovery.rs b/crates/temper-platform/src/os_apps/discovery.rs new file mode 100644 index 000000000..7d6fa4df5 --- /dev/null +++ b/crates/temper-platform/src/os_apps/discovery.rs @@ -0,0 +1,129 @@ +//! Deterministic discovery of OS app specifications and deployment policy. + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; + +use super::{AppDeploymentMode, AppManifest}; + +/// Finds all IOA spec files in an app directory. +/// +/// Root-level specs take precedence over duplicate filenames under `specs/`. +pub(super) fn find_ioa_files(app_dir: &Path) -> Vec<(String, PathBuf)> { + let mut results = Vec::new(); + let mut seen_names = HashSet::new(); + + scan_dir_for_ioa(app_dir, &mut results, &mut seen_names); + + let specs_dir = app_dir.join("specs"); + if specs_dir.is_dir() { + scan_dir_for_ioa(&specs_dir, &mut results, &mut seen_names); + } + + results +} + +fn scan_dir_for_ioa(dir: &Path, results: &mut Vec<(String, PathBuf)>, seen: &mut HashSet) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + let mut files: Vec<_> = entries + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.file_name().to_string_lossy().ends_with(".ioa.toml")) + .collect(); + files.sort_by_key(|entry| entry.file_name()); + + for entry in files { + let path = entry.path(); + let file_name = entry.file_name().to_string_lossy().to_string(); + if seen.insert(file_name) { + results.push((String::new(), path)); + } + } +} + +/// Finds the CSDL model file for an app, in deterministic precedence order. +pub(super) fn find_csdl(app_dir: &Path) -> Option { + let root = app_dir.join("model.csdl.xml"); + if root.exists() { + return Some(root); + } + + let specs = app_dir.join("specs").join("model.csdl.xml"); + if specs.exists() { + return Some(specs); + } + + let csdl_dir = app_dir.join("csdl"); + if !csdl_dir.is_dir() { + return None; + } + let Ok(entries) = std::fs::read_dir(csdl_dir) else { + return None; + }; + let mut files: Vec = entries + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.file_name().to_string_lossy().ends_with(".csdl.xml")) + .map(|entry| entry.path()) + .collect(); + files.sort(); + files.into_iter().next() +} + +/// Finds the app-owned Cedar policy files in deterministic order. +pub(super) fn find_cedar_policies(app_dir: &Path) -> Vec { + cedar_policy_files_in(&app_dir.join("policies")) +} + +/// Finds the Cedar commons policy files in deterministic order. +pub(super) fn find_commons_cedar_policies(app_dir: &Path) -> Vec { + cedar_policy_files_in(&app_dir.join("policies").join("commons")) +} + +fn cedar_policy_files_in(policies_dir: &Path) -> Vec { + if !policies_dir.is_dir() { + return Vec::new(); + } + let Ok(entries) = std::fs::read_dir(policies_dir) else { + return Vec::new(); + }; + let mut files: Vec = entries + .filter_map(|entry| entry.ok()) + .filter(|entry| entry.file_name().to_string_lossy().ends_with(".cedar")) + .map(|entry| entry.path()) + .collect(); + files.sort(); + files +} + +/// Returns an app-relative path with platform-independent separators. +pub(super) fn app_relative_path(app_dir: &Path, path: &Path) -> String { + path.strip_prefix(app_dir) + .unwrap_or(path) + .to_string_lossy() + .replace('\\', "/") +} + +/// Resolves the app deployment mode, honoring the documented environment overrides. +pub(super) fn effective_app_deployment_mode(manifest: &AppManifest) -> AppDeploymentMode { + let app_key = manifest + .name + .chars() + .map(|ch| { + if ch.is_ascii_alphanumeric() { + ch.to_ascii_uppercase() + } else { + '_' + } + }) + .collect::(); + let specific_key = format!("TEMPER_OS_APP_{app_key}_MODE"); + let raw = std::env::var(&specific_key) + .or_else(|_| std::env::var("TEMPER_OS_APP_MODE")) + .ok(); + + match raw.as_deref().map(str::trim) { + Some("commons") | Some("Commons") | Some("COMMONS") => AppDeploymentMode::Commons, + Some("operator") | Some("Operator") | Some("OPERATOR") => AppDeploymentMode::Operator, + _ => manifest.mode, + } +} diff --git a/crates/temper-platform/src/os_apps/entity_aliases.rs b/crates/temper-platform/src/os_apps/entity_aliases.rs index 270795a0e..08d1f4206 100644 --- a/crates/temper-platform/src/os_apps/entity_aliases.rs +++ b/crates/temper-platform/src/os_apps/entity_aliases.rs @@ -12,7 +12,7 @@ pub(super) async fn ensure_created_file_initialized( ) -> Result<(), String> { let response = state .server - .get_tenant_entity_state(tenant_id, "File", file_id) + .get_tenant_entity_state_in_generation(tenant_id, "File", file_id, agent_ctx) .await .map_err(|e| format!("failed to inspect File('{file_id}') status: {e}"))?; if response.state.status != "Created" { @@ -42,10 +42,11 @@ pub(super) async fn ensure_entity_field_aliases( entity_type: &str, entity_id: &str, aliases: serde_json::Value, + agent_ctx: &temper_server::request_context::AgentContext, ) -> Result<(), String> { let current = state .server - .get_tenant_entity_state(tenant_id, entity_type, entity_id) + .get_tenant_entity_state_in_generation(tenant_id, entity_type, entity_id, agent_ctx) .await .map_err(|e| format!("failed to inspect {entity_type}('{entity_id}') aliases: {e}"))?; @@ -64,12 +65,14 @@ pub(super) async fn ensure_entity_field_aliases( state .server - .update_tenant_entity_fields( + .update_tenant_entity_fields_in_generation( tenant_id, entity_type, entity_id, serde_json::Value::Object(updates), false, + None, + agent_ctx, ) .await .map(|_| ()) @@ -81,10 +84,11 @@ pub(super) async fn file_already_contains( tenant_id: &TenantId, file_id: &str, desired_hash: &str, + agent_ctx: &temper_server::request_context::AgentContext, ) -> Result { let response = state .server - .get_tenant_entity_state(tenant_id, "File", file_id) + .get_tenant_entity_state_in_generation(tenant_id, "File", file_id, agent_ctx) .await .map_err(|e| format!("failed to inspect File('{file_id}'): {e}"))?; diff --git a/crates/temper-platform/src/os_apps/install/finalization.rs b/crates/temper-platform/src/os_apps/install/finalization.rs new file mode 100644 index 000000000..043d3d565 --- /dev/null +++ b/crates/temper-platform/src/os_apps/install/finalization.rs @@ -0,0 +1,136 @@ +//! Terminal metadata finalization for a completed OS-app runtime generation. + +use std::future::Future; + +use super::super::*; + +pub(in crate::os_apps) async fn finalize_os_app_publication( + state: &PlatformState, + publication_guard: &mut SpecPublicationGuard, + tenant: &TenantId, + schedule_post_cutover_maintenance: P, + terminal_metadata_write: F, +) -> Result<(), String> +where + F: Future>, + P: FnOnce(), +{ + state + .server + .complete_spec_publication_retry(publication_guard, tenant)?; + schedule_post_cutover_maintenance(); + // The terminal `installed` row is a derived recovery marker, not part of + // the runtime-generation fingerprint. Release the exact armed generation + // before this cancellable, outcome-ambiguous acknowledgement: a retry can + // safely reconcile either the preceding `publishing` row or the committed + // terminal row without inheriting an unreconstructible sticky intent. + terminal_metadata_write.await +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn ambiguous_terminal_metadata_ack_cannot_strand_a_completed_runtime_generation() { + let maintenance_scheduled = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let state = PlatformState::new(None); + let tenant = TenantId::new("terminal-metadata-ambiguity"); + let mut publication = state + .server + .begin_spec_publication(&tenant) + .await + .expect("acquire publication writer"); + let intent = temper_server::ServerState::spec_publication_intent( + "terminal-metadata-test", + [("generation", b"complete".as_slice())], + ); + state + .server + .arm_spec_publication(&mut publication, &tenant, &intent) + .expect("arm runtime generation"); + + let maintenance_observer = std::sync::Arc::clone(&maintenance_scheduled); + let error = finalize_os_app_publication( + &state, + &mut publication, + &tenant, + move || { + maintenance_observer.store(true, std::sync::atomic::Ordering::SeqCst); + }, + async { Err("injected lost terminal metadata acknowledgement".to_string()) }, + ) + .await + .expect_err("terminal metadata acknowledgement remains visible to the caller"); + + assert!(error.contains("lost terminal metadata acknowledgement")); + assert!( + !state.server.spec_publication_gated(&tenant), + "terminal metadata is a derived completion marker and cannot retain runtime-generation debt" + ); + assert_eq!(state.server.tenant_generation_version(&tenant), 1); + assert!( + maintenance_scheduled.load(std::sync::atomic::Ordering::SeqCst), + "post-cutover maintenance must run before the ambiguous metadata await" + ); + } + + #[tokio::test] + async fn cancelled_terminal_metadata_ack_wait_cannot_strand_a_completed_runtime_generation() { + let maintenance_scheduled = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let state = PlatformState::new(None); + let tenant = TenantId::new("terminal-metadata-cancellation"); + let mut publication = state + .server + .begin_spec_publication(&tenant) + .await + .expect("acquire publication writer"); + let intent = temper_server::ServerState::spec_publication_intent( + "terminal-metadata-cancellation-test", + [("generation", b"complete".as_slice())], + ); + state + .server + .arm_spec_publication(&mut publication, &tenant, &intent) + .expect("arm runtime generation"); + let task_state = state.clone(); + let task_tenant = tenant.clone(); + let maintenance_observer = std::sync::Arc::clone(&maintenance_scheduled); + let (write_polled_tx, write_polled_rx) = tokio::sync::oneshot::channel(); + let task = tokio::spawn(async move { + finalize_os_app_publication( + &task_state, + &mut publication, + &task_tenant, + move || { + maintenance_observer.store(true, std::sync::atomic::Ordering::SeqCst); + }, + async move { + let _ = write_polled_tx.send(()); + std::future::pending::>().await + }, + ) + .await + }); + write_polled_rx + .await + .expect("terminal metadata write should begin"); + task.abort(); + assert!( + task.await + .expect_err("finalization task should cancel") + .is_cancelled(), + "test must cancel during the outcome-ambiguous terminal metadata await" + ); + + assert!( + !state.server.spec_publication_gated(&tenant), + "cancellation after a possible metadata commit cannot retain runtime-generation debt" + ); + assert_eq!(state.server.tenant_generation_version(&tenant), 1); + assert!( + maintenance_scheduled.load(std::sync::atomic::Ordering::SeqCst), + "post-cutover maintenance must be scheduled before cancellation is possible" + ); + } +} diff --git a/crates/temper-platform/src/os_apps/install/mod.rs b/crates/temper-platform/src/os_apps/install/mod.rs new file mode 100644 index 000000000..420bd46ea --- /dev/null +++ b/crates/temper-platform/src/os_apps/install/mod.rs @@ -0,0 +1,176 @@ +//! OS-app installation preflight and publication entry points. + +use super::*; + +mod finalization; +mod publication_intent; +mod uploaded_wasm; +mod wasm_activation; + +pub(super) use finalization::finalize_os_app_publication; +pub(super) use publication_intent::append_installed_app_record_intent; +pub(super) use uploaded_wasm::{UploadedWasmReplacementContext, uploaded_wasm_replacement_context}; +pub(super) use wasm_activation::activate_wasm_generation; + +/// Install an OS app into a tenant (workspace). +/// +/// Reads app files from disk, runs the verification cascade, registers +/// specs in the SpecRegistry, loads Cedar policies, and persists everything +/// to the platform DB so specs survive redeployments. +/// +/// Durable state is written before in-memory activation. A persistence error +/// therefore leaves the registry and Cedar engine on the prior generation. +pub async fn install_os_app( + state: &PlatformState, + tenant: &str, + app_name: &str, +) -> Result { + let order = resolve_os_app_install_order(&[app_name.to_string()])?; + + let mut final_result = None; + for app in order { + let result = install_os_app_without_dependencies(state, tenant, &app).await?; + if app == app_name { + final_result = Some(result); + } + } + final_result.ok_or_else(|| format!("OS app '{app_name}' produced no install result")) +} + +async fn install_os_app_without_dependencies( + state: &PlatformState, + tenant: &str, + app_name: &str, +) -> Result { + install_os_app_with_plan(state, tenant, app_name, OsAppInstallPlan::all()).await +} + +pub(super) fn bundled_wasm_replaces_existing( + module_name: &str, + wasm_bytes: &[u8], + bundle: &AppBundle, + existing_sources: &BTreeMap, + upload_replacement: &UploadedWasmReplacementContext, +) -> bool { + let hash = temper_wasm::WasmEngine::hash_module(wasm_bytes); + let required = bundle + .wasm_module_configs + .get(module_name) + .is_some_and(WasmModuleManifest::is_required); + existing_sources.get(module_name).is_some_and(|existing| { + upload_replacement.should_replace(existing) + || (required && existing.source == "upload" && existing.sha256_hash != hash) + }) +} + +#[expect( + clippy::too_many_arguments, + reason = "publication validation receives the complete preflight generation" +)] +pub(super) fn validate_os_app_publication_candidate( + state: &PlatformState, + tenant: &str, + app_name: &str, + plan: OsAppInstallPlan, + bundle: &AppBundle, + complete_specs: &BTreeMap, + complete_csdl: Option<&str>, + complete_constraints: Option<&str>, + complete_policy: &str, + existing_sources: &BTreeMap, + upload_replacement: &UploadedWasmReplacementContext, +) -> Result<(), String> { + temper_authz::AuthzEngine::new(complete_policy).map_err(|error| { + format!("Invalid merged Cedar policies for os-app '{app_name}': {error}") + })?; + + if !complete_specs.is_empty() { + let csdl_source = complete_csdl.ok_or_else(|| { + format!("OS app '{app_name}' publishes specs without a complete tenant CSDL") + })?; + let csdl = parse_csdl(csdl_source) + .map_err(|error| format!("Invalid merged CSDL for os-app '{app_name}': {error}"))?; + let specs = complete_specs + .iter() + .map(|(entity_type, source)| (entity_type.as_str(), source.as_str())) + .collect::>(); + let mut candidate_registry = temper_server::registry::SpecRegistry::new(); + candidate_registry + .try_register_tenant_with_reactions_constraints_and_key_epochs( + TenantId::new(tenant), + csdl, + csdl_source.to_string(), + &specs, + Vec::new(), + complete_constraints.map(str::to_string), + false, + &BTreeMap::new(), + ) + .map_err(|error| { + format!("Invalid complete spec generation for os-app '{app_name}': {error}") + })?; + } else if let Some(source) = complete_constraints { + temper_spec::cross_invariant::parse_cross_invariants(source).map_err(|error| { + format!("Invalid cross-invariants for os-app '{app_name}': {error}") + })?; + } + + if plan.wasm { + for (module_name, config) in &bundle.wasm_module_configs { + if config.is_required() && !bundle.wasm_modules.contains_key(module_name) { + return Err(format!( + "Required WASM module '{module_name}' is missing from os-app '{app_name}'" + )); + } + } + for (module_name, wasm_bytes) in &bundle.wasm_modules { + let hash = temper_wasm::WasmEngine::hash_module(wasm_bytes); + let preserve_upload = existing_sources.get(module_name).is_some_and(|existing| { + existing.source == "upload" + && existing.sha256_hash != hash + && !bundled_wasm_replaces_existing( + module_name, + wasm_bytes, + bundle, + existing_sources, + upload_replacement, + ) + }); + let config = bundle.wasm_module_configs.get(module_name); + let must_compile = config.is_some_and(|config| { + config.is_required() || config.startup_loading == WasmStartupLoading::Eager + }); + if must_compile && !preserve_upload { + state + .server + .wasm_engine + .compile_and_cache(wasm_bytes) + .map_err(|error| { + format!( + "Invalid required/eager WASM module '{module_name}' in os-app '{app_name}': {error}" + ) + })?; + } + } + } + Ok(()) +} + +pub(super) async fn install_os_app_with_plan( + state: &PlatformState, + tenant: &str, + app_name: &str, + plan: OsAppInstallPlan, +) -> Result { + let tenant_id = TenantId::new(tenant); + let mut publication_guard = state.server.begin_spec_publication(&tenant_id).await?; + install_os_app_with_plan_under_guard( + state, + tenant, + app_name, + plan, + None, + &mut publication_guard, + ) + .await +} diff --git a/crates/temper-platform/src/os_apps/install/publication_intent.rs b/crates/temper-platform/src/os_apps/install/publication_intent.rs new file mode 100644 index 000000000..d2387bbbd --- /dev/null +++ b/crates/temper-platform/src/os_apps/install/publication_intent.rs @@ -0,0 +1,30 @@ +use super::*; + +pub(in crate::os_apps) fn append_installed_app_record_intent( + components: &mut Vec<(String, Vec)>, + record: &InstalledAppRecord, +) { + for (name, value) in [ + ("tenant", record.tenant.as_str()), + ("app_name", record.app_name.as_str()), + ("source_kind", record.source_kind.as_str()), + ("app_ref", record.app_ref.as_str()), + ("version_hash", record.version_hash.as_str()), + ("pinned_version_hash", record.pinned_version_hash.as_str()), + ("current_version_hash", record.current_version_hash.as_str()), + ("follow_policy", record.follow_policy.as_str()), + ("closure_id", record.closure_id.as_str()), + ("registry_url", record.registry_url.as_str()), + ("registry_tenant", record.registry_tenant.as_str()), + ("app_version", record.app_version.as_str()), + ("bundle_digest", record.bundle_digest.as_str()), + ("spec_digest", record.spec_digest.as_str()), + ("policy_digest", record.policy_digest.as_str()), + ("wasm_digest", record.wasm_digest.as_str()), + ("content_digest", record.content_digest.as_str()), + ("seed_digest", record.seed_digest.as_str()), + ("status", record.status.as_str()), + ] { + components.push((format!("installed_app:{name}"), value.as_bytes().to_vec())); + } +} diff --git a/crates/temper-platform/src/os_apps/install/uploaded_wasm.rs b/crates/temper-platform/src/os_apps/install/uploaded_wasm.rs new file mode 100644 index 000000000..af05c689e --- /dev/null +++ b/crates/temper-platform/src/os_apps/install/uploaded_wasm.rs @@ -0,0 +1,95 @@ +//! Source-aware replacement policy for hot-uploaded WASM modules. + +use chrono::{DateTime, NaiveDateTime, Utc}; +use serde::{Deserialize, Serialize}; + +use super::super::*; + +#[derive(Debug, Clone, Default, Deserialize, Serialize)] +pub(in crate::os_apps) struct UploadedWasmReplacementContext { + bundle_wasm_digest_changed: bool, + app_recorded_at: Option>, +} + +impl UploadedWasmReplacementContext { + pub(in crate::os_apps) fn publication_status(&self) -> String { + match serde_json::to_string(self) { + Ok(context) => format!("publishing:{context}"), + Err(error) => { + tracing::error!(%error, "Failed to serialize WASM publication context"); + "publishing".to_string() + } + } + } + + fn from_publication_status(status: &str) -> Option { + serde_json::from_str(status.strip_prefix("publishing:")?).ok() + } + + pub(in crate::os_apps) fn should_replace(&self, existing: &WasmModuleSource) -> bool { + if existing.source != "upload" { + return false; + } + if self.bundle_wasm_digest_changed { + return true; + } + let Some(app_recorded_at) = self.app_recorded_at else { + return false; + }; + let Some(updated_at) = existing + .updated_at + .as_deref() + .and_then(parse_persisted_timestamp) + else { + return false; + }; + updated_at < app_recorded_at + } +} + +fn parse_persisted_timestamp(value: &str) -> Option> { + DateTime::parse_from_rfc3339(value) + .map(|dt| dt.with_timezone(&Utc)) + .or_else(|_| { + NaiveDateTime::parse_from_str(value, "%Y-%m-%d %H:%M:%S").map(|dt| dt.and_utc()) + }) + .ok() +} + +pub(in crate::os_apps) async fn uploaded_wasm_replacement_context( + state: &PlatformState, + tenant: &str, + app_name: &str, + bundle: &AppBundle, +) -> UploadedWasmReplacementContext { + let Some(ps) = state + .server + .storage_stack + .as_ref() + .and_then(|stack| stack.platform.clone()) + else { + return UploadedWasmReplacementContext::default(); + }; + let digest = reconcile::digest_app_bundle(app_name, bundle); + match ps.get_installed_app(tenant, app_name).await { + Ok(Some(record)) => UploadedWasmReplacementContext::from_publication_status(&record.status) + .unwrap_or_else(|| UploadedWasmReplacementContext { + bundle_wasm_digest_changed: record.wasm_digest != digest.wasm_digest, + app_recorded_at: record + .last_reconciled_at + .as_deref() + .or(record.installed_at.as_deref()) + .and_then(parse_persisted_timestamp), + }), + Ok(None) => UploadedWasmReplacementContext::default(), + Err(error) => { + tracing::warn!( + tenant, + app = %app_name, + error = %error, + "Failed to read OS app metadata while deciding WASM hot-upload replacement" + ); + UploadedWasmReplacementContext::default() + } + } +} diff --git a/crates/temper-platform/src/os_apps/install/wasm_activation.rs b/crates/temper-platform/src/os_apps/install/wasm_activation.rs new file mode 100644 index 000000000..bb7b29534 --- /dev/null +++ b/crates/temper-platform/src/os_apps/install/wasm_activation.rs @@ -0,0 +1,158 @@ +//! In-memory activation for the durable WASM generation selected at preflight. + +use std::time::Instant; + +use super::super::*; + +pub(in crate::os_apps) struct WasmActivation { + pub(in crate::os_apps) registered: Vec, + pub(in crate::os_apps) skipped: Vec, + pub(in crate::os_apps) failures: Vec, +} + +#[expect( + clippy::too_many_arguments, + reason = "WASM activation consumes one complete validated installation generation" +)] +pub(in crate::os_apps) async fn activate_wasm_generation( + state: &PlatformState, + tenant: &str, + app_name: &str, + plan: OsAppInstallPlan, + bundle: &AppBundle, + tenant_id: &TenantId, + effective_wasm_hashes: &BTreeMap, + existing_sources: &BTreeMap, + upload_replacement: &UploadedWasmReplacementContext, +) -> Result { + let mut registered = Vec::new(); + let mut skipped = Vec::new(); + let mut failures = Vec::new(); + if plan.wasm { + for (module_name, wasm_bytes) in &bundle.wasm_modules { + let module_started = Instant::now(); + let module_config = bundle.wasm_module_configs.get(module_name); + let bundled_hash = temper_wasm::WasmEngine::hash_module(wasm_bytes); + let hash = effective_wasm_hashes + .get(module_name) + .cloned() + .unwrap_or_else(|| bundled_hash.clone()); + let replace_uploaded_module = bundled_wasm_replaces_existing( + module_name, + wasm_bytes, + bundle, + existing_sources, + upload_replacement, + ); + if let Some(existing) = existing_sources.get(module_name) + && existing.source == "upload" + && existing.sha256_hash != bundled_hash + { + if replace_uploaded_module { + tracing::info!( + tenant, + module = %module_name, + bundled_hash = %hash, + upload_hash = %existing.sha256_hash, + "Replacing stale hot-uploaded WASM module during os-app reconcile" + ); + } else { + tracing::info!( + tenant, + module = %module_name, + bundled_hash = %hash, + upload_hash = %existing.sha256_hash, + "Skipping bundled install: hot-uploaded module preserved" + ); + state + .server + .wasm_module_registry + .write() + .map_err(|error| { + format!("WASM registry lock poisoned during os-app install: {error}") + })? + .register(tenant_id, module_name, &hash); + skipped.push(module_name.clone()); + continue; + } + } + state + .server + .wasm_module_registry + .write() + .map_err(|error| { + format!("WASM registry lock poisoned during os-app install: {error}") + })? + .register(tenant_id, module_name, &hash); + + if matches!( + module_config.map(|config| config.startup_loading), + Some(WasmStartupLoading::Eager) + ) && let Err(error) = state.server.wasm_engine.compile_and_cache(wasm_bytes) + { + failures.push(module_name.clone()); + tracing::warn!( + tenant, + module = %module_name, + error = %error, + "Failed to eagerly compile WASM module from OS app" + ); + } + + tracing::info!( + tenant, + module = %module_name, + hash = %hash, + size = wasm_bytes.len(), + duration_ms = module_started.elapsed().as_millis() as u64, + startup_loading = ?module_config + .map(|config| config.startup_loading) + .unwrap_or_default(), + "WASM module registered from OS app" + ); + registered.push(module_name.clone()); + } + + for (module_name, module_config) in &bundle.wasm_module_configs { + if bundle.wasm_modules.contains_key(module_name) { + continue; + } + match module_config.criticality { + WasmModuleCriticality::Optional => { + skipped.push(module_name.clone()); + tracing::warn!( + tenant, + module = %module_name, + "Configured optional WASM module artifact is missing from the app bundle" + ); + } + WasmModuleCriticality::PlatformRequired | WasmModuleCriticality::AppRequired => { + failures.push(module_name.clone()); + tracing::error!( + tenant, + module = %module_name, + criticality = ?module_config.criticality, + "Configured required WASM module artifact is missing from the app bundle" + ); + } + } + } + } + + if !failures.is_empty() { + failures.sort(); + failures.dedup(); + return Err(format!( + "WASM modules named by the os-app generation were not activated for '{app_name}': {}", + failures.join(", ") + )); + } + if bundle.deployment_mode == AppDeploymentMode::Commons { + state.server.enable_commons_guardrails(tenant); + } + Ok(WasmActivation { + registered, + skipped, + failures, + }) +} diff --git a/crates/temper-platform/src/os_apps/mod.rs b/crates/temper-platform/src/os_apps/mod.rs index b1e0b78ad..48cc0f669 100644 --- a/crates/temper-platform/src/os_apps/mod.rs +++ b/crates/temper-platform/src/os_apps/mod.rs @@ -8,12 +8,14 @@ use std::collections::BTreeMap; use std::path::{Path, PathBuf}; -use std::time::Instant; -use chrono::{DateTime, NaiveDateTime, Utc}; use serde::Serialize; use temper_runtime::tenant::TenantId; -use temper_server::state::WasmModuleSource; +use temper_server::platform_store::{ + InstalledAppRecord, OsAppPublication, PolicyEntryPublication, SpecPublication, + SpecPublicationMode, TenantConstraintsPublication, TenantPolicyPublication, WasmPublication, +}; +use temper_server::state::{SpecPublicationGuard, WasmModuleSource}; use temper_spec::automaton; use temper_spec::csdl::{emit_csdl_xml, merge_csdl, parse_csdl}; @@ -23,7 +25,9 @@ use crate::state::PlatformState; mod agent_bootstrap; mod app_catalog; mod closure_bootstrap; +mod discovery; mod entity_aliases; +mod install; mod policy_rows; mod reconcile; mod runtime_heal; @@ -39,15 +43,28 @@ pub use closure_bootstrap::{ bootstrap_closure_manifest, os_app_closure_for_roots, parse_bootstrap_manifest_str, startup_os_app_closure, }; +use discovery::{ + app_relative_path, effective_app_deployment_mode, find_cedar_policies, + find_commons_cedar_policies, find_csdl, find_ioa_files, +}; use entity_aliases::{ content_sha256, ensure_created_file_initialized, ensure_entity_field_aliases, file_already_contains, slug_fragment, }; +pub use install::install_os_app; +#[cfg(test)] +use install::install_os_app_with_plan; +use install::{ + UploadedWasmReplacementContext, activate_wasm_generation, append_installed_app_record_intent, + bundled_wasm_replaces_existing, finalize_os_app_publication, uploaded_wasm_replacement_context, + validate_os_app_publication_candidate, +}; #[cfg(test)] pub(super) use policy_rows::os_app_policy_row_id; pub use reconcile::{os_app_bundle_digest, reconcile_os_app, resolve_os_app_install_order}; pub(crate) use reconcile::{ - tenant_has_active_policies_for_bundle, tenant_has_registered_wasm_for_bundle, + reconcile_os_app_under_guard, tenant_has_active_policies_for_bundle, + tenant_has_registered_wasm_for_bundle, }; pub(crate) use runtime_heal::{ restore_app_specs_from_matching_digest, tenant_has_ready_app_specs_for_bundle, @@ -201,153 +218,6 @@ struct SeedFile { pub type OsAppEntry = AppEntry; pub type OsAppBundle = AppBundle; -/// Find all IOA spec files in an app directory. -/// -/// Handles both layouts: -/// - Root-level: `app-name/*.ioa.toml` + `app-name/model.csdl.xml` -/// - Specs subdir: `app-name/specs/*.ioa.toml` + `app-name/specs/model.csdl.xml` -/// -/// Returns `(entity_type_hint, path)` pairs. The entity type is extracted -/// from the IOA file's `[automaton] name` field, not the filename. -fn find_ioa_files(app_dir: &Path) -> Vec<(String, PathBuf)> { - let mut results = Vec::new(); - let mut seen_names = std::collections::HashSet::new(); - - // Scan root first (takes priority for dedup). - scan_dir_for_ioa(app_dir, &mut results, &mut seen_names); - - // Then scan specs/ subdirectory. - let specs_dir = app_dir.join("specs"); - if specs_dir.is_dir() { - scan_dir_for_ioa(&specs_dir, &mut results, &mut seen_names); - } - - results -} - -/// Scan a single directory for `*.ioa.toml` files. -fn scan_dir_for_ioa( - dir: &Path, - results: &mut Vec<(String, PathBuf)>, - seen: &mut std::collections::HashSet, -) { - let Ok(entries) = std::fs::read_dir(dir) else { - return; - }; - let mut files: Vec<_> = entries - .filter_map(|e| e.ok()) - .filter(|e| e.file_name().to_string_lossy().ends_with(".ioa.toml")) - .collect(); - files.sort_by_key(|e| e.file_name()); - - for entry in files { - let path = entry.path(); - let fname = entry.file_name().to_string_lossy().to_string(); - // Use filename as dedup key. - if !seen.insert(fname) { - continue; - } - results.push((String::new(), path)); - } -} - -/// Find the CSDL model file in an app directory. -fn find_csdl(app_dir: &Path) -> Option { - // Root-level first. - let root = app_dir.join("model.csdl.xml"); - if root.exists() { - return Some(root); - } - // Then specs/. - let specs = app_dir.join("specs").join("model.csdl.xml"); - if specs.exists() { - return Some(specs); - } - // Then a dedicated csdl/ directory. - let csdl_dir = app_dir.join("csdl"); - if csdl_dir.is_dir() { - let Ok(entries) = std::fs::read_dir(&csdl_dir) else { - return None; - }; - let mut files: Vec = entries - .filter_map(|e| e.ok()) - .filter(|e| e.file_name().to_string_lossy().ends_with(".csdl.xml")) - .map(|e| e.path()) - .collect(); - files.sort(); - if let Some(first) = files.into_iter().next() { - return Some(first); - } - } - None -} - -/// Find all Cedar policy files in an app directory. -fn find_cedar_policies(app_dir: &Path) -> Vec { - let policies_dir = app_dir.join("policies"); - if !policies_dir.is_dir() { - return Vec::new(); - } - let Ok(entries) = std::fs::read_dir(&policies_dir) else { - return Vec::new(); - }; - let mut files: Vec = entries - .filter_map(|e| e.ok()) - .filter(|e| e.file_name().to_string_lossy().ends_with(".cedar")) - .map(|e| e.path()) - .collect(); - files.sort(); - files -} - -fn find_commons_cedar_policies(app_dir: &Path) -> Vec { - let policies_dir = app_dir.join("policies").join("commons"); - if !policies_dir.is_dir() { - return Vec::new(); - } - let Ok(entries) = std::fs::read_dir(&policies_dir) else { - return Vec::new(); - }; - let mut files: Vec = entries - .filter_map(|e| e.ok()) - .filter(|e| e.file_name().to_string_lossy().ends_with(".cedar")) - .map(|e| e.path()) - .collect(); - files.sort(); - files -} - -fn app_relative_path(app_dir: &Path, path: &Path) -> String { - path.strip_prefix(app_dir) - .unwrap_or(path) - .to_string_lossy() - .replace('\\', "/") -} - -fn effective_app_deployment_mode(manifest: &AppManifest) -> AppDeploymentMode { - let app_key = manifest - .name - .chars() - .map(|ch| { - if ch.is_ascii_alphanumeric() { - ch.to_ascii_uppercase() - } else { - '_' - } - }) - .collect::(); - let specific_key = format!("TEMPER_OS_APP_{app_key}_MODE"); - let raw = std::env::var(&specific_key) - .or_else(|_| std::env::var("TEMPER_OS_APP_MODE")) - .ok(); - - match raw.as_deref().map(str::trim) { - Some("commons") | Some("Commons") | Some("COMMONS") => AppDeploymentMode::Commons, - Some("operator") | Some("Operator") | Some("OPERATOR") => AppDeploymentMode::Operator, - _ => manifest.mode, - } -} - /// Find compiled WASM module binaries in an app directory. /// /// Scans packaged artifacts copied next to the module first, then falls back @@ -1064,45 +934,13 @@ fn local_os_app_dependency_name(dependency: &str) -> Option { } } -/// Install an OS app into a tenant (workspace). -/// -/// Reads app files from disk, runs the verification cascade, registers -/// specs in the SpecRegistry, loads Cedar policies, and **persists -/// everything to the platform DB** so specs survive redeployments. -/// -/// **Write ordering:** Turso first, then memory. If Turso persistence fails -/// the operation returns an error *before* touching in-memory state, so the -/// registry and Cedar engine stay consistent with the durable store. -pub async fn install_os_app( - state: &PlatformState, - tenant: &str, - app_name: &str, -) -> Result { - let order = resolve_os_app_install_order(&[app_name.to_string()])?; - - let mut final_result = None; - for app in order { - let result = install_os_app_without_dependencies(state, tenant, &app).await?; - if app == app_name { - final_result = Some(result); - } - } - final_result.ok_or_else(|| format!("OS app '{app_name}' produced no install result")) -} - -async fn install_os_app_without_dependencies( - state: &PlatformState, - tenant: &str, - app_name: &str, -) -> Result { - install_os_app_with_plan(state, tenant, app_name, OsAppInstallPlan::all()).await -} - -pub(super) async fn install_os_app_with_plan( +pub(super) async fn install_os_app_with_plan_under_guard( state: &PlatformState, tenant: &str, app_name: &str, plan: OsAppInstallPlan, + publication_record_override: Option, + publication_guard: &mut SpecPublicationGuard, ) -> Result { let app_dir = { let cat = catalog().read().unwrap(); // ci-ok: infallible lock @@ -1117,9 +955,6 @@ pub(super) async fn install_os_app_with_plan( app_dir.display() ) })?; - if bundle.deployment_mode == AppDeploymentMode::Commons { - state.server.enable_commons_guardrails(tenant); - } let tenant_id = TenantId::new(tenant); let upload_replacement = if plan.wasm && !bundle.wasm_modules.is_empty() { uploaded_wasm_replacement_context(state, tenant, app_name, &bundle).await @@ -1138,7 +973,7 @@ pub(super) async fn install_os_app_with_plan( // Classify each bundle spec as added / updated / skipped, and compute the // merged CSDL only when the reconcile plan needs spec work. let (mut added, mut updated, mut skipped, merged_csdl) = if plan.specs { - let registry = state.registry.read().unwrap(); // ci-ok: infallible lock + let registry = state.registry.read().expect("registry lock poisoned"); let mut added = Vec::new(); let mut updated = Vec::new(); let mut skipped = Vec::new(); @@ -1186,105 +1021,345 @@ pub(super) async fn install_os_app_with_plan( updated.sort(); skipped.sort(); - // Build the full Cedar policy text for this tenant (existing + new). - let combined_policy = if plan.policies && !bundle.cedar_policies.is_empty() { - let existing = cached_or_active_tenant_policy_text(state, tenant); - Some(merge_bundle_policies(&existing, &bundle.cedar_policies)) + let activation_spec_sources = bundle + .specs + .iter() + .map(|(entity_type, source)| (entity_type.as_str(), source.as_str())) + .collect::>(); + // Build the complete policy generation from durable granular rows while + // replacing this app's owned rows as one transaction. The aggregate blob + // remains a legacy/read cache; granular rows are the canonical source once + // any exist. + let mut atomic_policy_entries = Vec::new(); + let policy_owner = format!("os-app:{app_name}"); + let combined_policy = if plan.policies { + let existing_policy = cached_or_active_tenant_policy_text(state, tenant); + let durable_rows = if let Some(store) = state + .server + .storage_stack + .as_ref() + .and_then(|stack| stack.platform.clone()) + { + store.load_policy_entries().await.map_err(|error| { + format!("Failed to load durable Cedar policy rows for '{app_name}': {error}") + })? + } else { + Vec::new() + }; + let mut complete_rows = durable_rows + .iter() + .filter(|row| row.tenant == tenant && row.enabled) + .filter(|row| row.created_by != policy_owner) + .map(|row| (row.policy_id.clone(), row.cedar_text.clone())) + .collect::>(); + let tenant_has_granular_rows = durable_rows.iter().any(|row| row.tenant == tenant); + if !tenant_has_granular_rows && !existing_policy.trim().is_empty() { + atomic_policy_entries.push(policy_rows::OwnedPolicyEntry { + policy_id: "primary".to_string(), + cedar_text: existing_policy.trim().to_string(), + created_by: "legacy-migration".to_string(), + }); + complete_rows.insert("primary".to_string(), existing_policy.trim().to_string()); + } + let bundle_entries = policy_rows::bundle_policy_entries(app_name, &bundle); + for entry in &bundle_entries { + complete_rows.insert(entry.policy_id.clone(), entry.cedar_text.clone()); + } + atomic_policy_entries.extend(bundle_entries); + Some(merge_bundle_policies( + "", + &complete_rows.into_values().collect::>(), + )) } else { None }; - // ── Step 1: Persist to Turso FIRST (if available). ────────────── - // If any write fails, bail before touching in-memory state. - if let Some(turso) = state - .server - .storage_stack - .as_ref() - .and_then(|stack| stack.turso.as_ref()) - .and_then(|provider| provider.platform_store()) - { - if plan.specs - && let Some(ref merged) = merged_csdl - { - // Collect only this app's specs with their content hashes for a - // single transactional write. Existing tenant specs are preserved by - // the merged CSDL, but no longer re-written on every app install. - let owned: Vec<(String, String, String, String)> = bundle - .specs - .iter() - .map(|(et, ioa)| { - let hash = temper_store_turso::spec_content_hash(ioa); - (et.clone(), ioa.clone(), merged.clone(), hash) - }) - .collect(); - let refs: Vec<(&str, &str, &str, &str)> = owned - .iter() - .map(|(et, ioa, csdl, h)| (et.as_str(), ioa.as_str(), csdl.as_str(), h.as_str())) - .collect(); - turso - .upsert_specs_and_commit(tenant, &refs, combined_policy.as_deref(), app_name) - .await - .map_err(|e| format!("Failed to persist and commit specs: {e}"))?; - } else if let Some(ref policy_text) = combined_policy { - turso - .upsert_tenant_policy(tenant, policy_text) - .await - .map_err(|e| format!("Failed to persist Cedar policy: {e}"))?; - turso - .record_installed_app(tenant, app_name) - .await - .map_err(|e| format!("Failed to record os-app installation: {e}"))?; + let existing_policy = cached_or_active_tenant_policy_text(state, tenant); + let complete_policy = combined_policy.as_deref().unwrap_or(&existing_policy); + let (complete_specs, complete_csdl, complete_constraints) = { + let registry = state.registry.read().unwrap(); // ci-ok: infallible lock + let existing = registry.get_tenant(&tenant_id); + let mut specs = existing + .map(|config| { + config + .entities + .iter() + .map(|(entity_type, spec)| (entity_type.clone(), spec.ioa_source.clone())) + .collect::>() + }) + .unwrap_or_default(); + if plan.specs { + specs.extend(bundle.specs.iter().cloned()); } - if plan.specs - && let Some(ref cross_invariants_toml) = bundle.cross_invariants_toml - { - turso - .upsert_tenant_constraints(tenant, cross_invariants_toml) - .await - .map_err(|e| format!("Failed to persist cross-invariants: {e}"))?; + let csdl = merged_csdl + .clone() + .or_else(|| existing.map(|config| config.csdl_xml.as_ref().clone())); + let constraints = if plan.specs { + bundle + .cross_invariants_toml + .clone() + .or_else(|| existing.and_then(|config| config.cross_invariants_source.clone())) + } else { + existing.and_then(|config| config.cross_invariants_source.clone()) + }; + (specs, csdl, constraints) + }; + let existing_sources = + if !bundle.wasm_modules.is_empty() || !bundle.wasm_module_configs.is_empty() { + state.server.load_wasm_module_sources(tenant).await? + } else { + BTreeMap::new() + }; + validate_os_app_publication_candidate( + state, + tenant, + app_name, + plan, + &bundle, + &complete_specs, + complete_csdl.as_deref(), + complete_constraints.as_deref(), + complete_policy, + &existing_sources, + &upload_replacement, + )?; + + let mut effective_wasm_hashes = BTreeMap::new(); + let mut durable_wasm_publications = Vec::new(); + if plan.wasm { + for (module_name, wasm_bytes) in &bundle.wasm_modules { + let bundled_hash = temper_wasm::WasmEngine::hash_module(wasm_bytes); + let replace_uploaded_module = bundled_wasm_replaces_existing( + module_name, + wasm_bytes, + &bundle, + &existing_sources, + &upload_replacement, + ); + let preserve_upload = existing_sources.get(module_name).is_some_and(|existing| { + existing.source == "upload" + && existing.sha256_hash != bundled_hash + && !replace_uploaded_module + }); + if preserve_upload { + let existing = existing_sources + .get(module_name) + .expect("preserved upload source checked above"); + effective_wasm_hashes.insert(module_name.clone(), existing.sha256_hash.clone()); + } else { + effective_wasm_hashes.insert(module_name.clone(), bundled_hash.clone()); + durable_wasm_publications.push(( + module_name.as_str(), + wasm_bytes.as_slice(), + bundled_hash, + if replace_uploaded_module { + "bundled-replace-upload" + } else { + "bundled" + }, + )); + } + } + } + let durable_wasm_publication_refs = durable_wasm_publications + .iter() + .map( + |(module_name, wasm_bytes, sha256_hash, source)| WasmPublication { + module_name, + wasm_bytes, + sha256_hash, + source, + }, + ) + .collect::>(); + + let bundle_digest = reconcile::digest_app_bundle(app_name, &bundle); + let default_publication_record = InstalledAppRecord { + tenant: tenant.to_string(), + app_name: app_name.to_string(), + source_kind: "local".to_string(), + app_ref: String::new(), + version_hash: String::new(), + pinned_version_hash: String::new(), + current_version_hash: String::new(), + follow_policy: "pinned".to_string(), + closure_id: String::new(), + registry_url: String::new(), + registry_tenant: String::new(), + app_version: bundle_digest.app_version.clone(), + bundle_digest: bundle_digest.bundle_digest.clone(), + spec_digest: bundle_digest.spec_digest.clone(), + policy_digest: bundle_digest.policy_digest.clone(), + wasm_digest: bundle_digest.wasm_digest.clone(), + content_digest: bundle_digest.content_digest.clone(), + seed_digest: bundle_digest.seed_digest.clone(), + installed_at: None, + last_reconciled_at: None, + status: upload_replacement.publication_status(), + }; + let publication_record = match publication_record_override { + Some(mut record) => { + if record.tenant != tenant || record.app_name != app_name { + return Err(format!( + "OS app publication provenance identifies {}:{} while installing {tenant}:{app_name}", + record.tenant, record.app_name + )); + } + record.app_version = bundle_digest.app_version.clone(); + record.bundle_digest = bundle_digest.bundle_digest.clone(); + record.spec_digest = bundle_digest.spec_digest.clone(); + record.policy_digest = bundle_digest.policy_digest.clone(); + record.wasm_digest = bundle_digest.wasm_digest.clone(); + record.content_digest = bundle_digest.content_digest.clone(); + record.seed_digest = bundle_digest.seed_digest.clone(); + record.status = upload_replacement.publication_status(); + record } - } else if let Some(ps) = state + None => default_publication_record, + }; + let mut intent_components = vec![ + ( + "bundle".to_string(), + bundle_digest.bundle_digest.as_bytes().to_vec(), + ), + ( + "csdl".to_string(), + complete_csdl.as_deref().unwrap_or("").as_bytes().to_vec(), + ), + ( + "constraints".to_string(), + complete_constraints + .as_deref() + .unwrap_or("") + .as_bytes() + .to_vec(), + ), + ("policy".to_string(), complete_policy.as_bytes().to_vec()), + ( + "commons_guardrails".to_string(), + (state.server.commons_guardrails_enabled(&tenant_id) + || bundle.deployment_mode == AppDeploymentMode::Commons) + .to_string() + .into_bytes(), + ), + ]; + append_installed_app_record_intent(&mut intent_components, &publication_record); + intent_components.extend( + complete_specs.iter().map(|(entity_type, source)| { + (format!("spec:{entity_type}"), source.as_bytes().to_vec()) + }), + ); + for (module_name, effective_hash) in &effective_wasm_hashes { + intent_components.push(( + format!("wasm:{module_name}"), + effective_hash.as_bytes().to_vec(), + )); + } + let intent_refs = intent_components + .iter() + .map(|(name, value)| (name.as_str(), value.as_slice())) + .collect::>(); + let publication_intent = temper_server::ServerState::spec_publication_intent( + "os-app-runtime-generation", + intent_refs, + ); + + // ── Step 1: Persist before touching in-memory state. ───────────── + state + .server + .arm_spec_publication(publication_guard, &tenant_id, &publication_intent)?; + if let Some(ps) = state .server .storage_stack .as_ref() .and_then(|stack| stack.platform.clone()) { - if plan.specs - && let Some(ref merged) = merged_csdl - { - for (entity_type, ioa_source) in &bundle.specs { - let hash = temper_store_turso::spec_content_hash(ioa_source); - ps.upsert_spec(tenant, entity_type, ioa_source, merged, &hash) - .await - .map_err(|e| format!("Failed to persist spec {entity_type}: {e}"))?; - } - } - if let Some(ref policy_text) = combined_policy { - ps.upsert_tenant_policy(tenant, policy_text) - .await - .map_err(|e| format!("Failed to persist Cedar policy: {e}"))?; - } - if plan.specs - && let Some(ref cross_invariants_toml) = bundle.cross_invariants_toml - { - ps.upsert_tenant_constraints(tenant, cross_invariants_toml) - .await - .map_err(|e| format!("Failed to persist cross-invariants: {e}"))?; - } - ps.record_installed_app(tenant, app_name) + let publications = if plan.specs { + bundle + .specs + .iter() + .map(|(entity_type, ioa_source)| { + ( + entity_type, + ioa_source, + temper_store_turso::spec_content_hash(ioa_source), + ) + }) + .collect::>() + } else { + Vec::new() + }; + let publication_refs = publications + .iter() + .map(|(entity_type, ioa_source, content_hash)| SpecPublication { + entity_type, + ioa_source, + csdl_xml: merged_csdl.as_deref().unwrap_or(""), + content_hash, + }) + .collect::>(); + let policy_entry_refs = atomic_policy_entries + .iter() + .map(|entry| PolicyEntryPublication { + policy_id: &entry.policy_id, + cedar_text: &entry.cedar_text, + created_by: &entry.created_by, + }) + .collect::>(); + let constraints = if plan.specs { + bundle + .cross_invariants_toml + .as_deref() + .map(|source| TenantConstraintsPublication::Replace(Some(source))) + .unwrap_or(TenantConstraintsPublication::Preserve) + } else { + TenantConstraintsPublication::Preserve + }; + if let Err(error) = ps + .publish_specs( + tenant, + &publication_refs, + SpecPublicationMode::Merge, + constraints, + combined_policy + .as_deref() + .map(TenantPolicyPublication::Replace) + .unwrap_or(TenantPolicyPublication::Preserve), + Some(OsAppPublication { + record: &publication_record, + policy_owner: plan.policies.then_some(policy_owner.as_str()), + policy_entries: &policy_entry_refs, + }), + None, + &durable_wasm_publication_refs, + ) .await - .map_err(|e| format!("Failed to record os-app installation: {e}"))?; - if plan.specs { - // Commit only when this path used individual spec writes. - ps.commit_specs(tenant) - .await - .map_err(|e| format!("Failed to commit specs: {e}"))?; + { + return Err(format!( + "Failed to atomically publish os-app bundle: {error}" + )); } } - if plan.policies { - policy_rows::persist_bundle_policy_rows(state, tenant, app_name, &bundle).await?; - } + // Durable spec publication is the crash boundary. Any returned error can be + // outcome-ambiguous, so the armed tenant remains fail-closed until a retry + // or startup recovery finishes. Only after acknowledged persistence do we + // atomically activate/purge the key contract and attach its epoch to the + // replacement live table. + let mut key_contract_cutover = if plan.specs { + Some( + state + .server + .prepare_key_index_contracts_for_spec_activation( + publication_guard, + &tenant_id, + &activation_spec_sources, + ) + .await?, + ) + } else { + None + }; // ── Step 2: Bootstrap into memory (verification + registry). ──── // Only process specs whose content has changed (added or updated); @@ -1334,6 +1409,12 @@ pub(super) async fn install_os_app_with_plan( verified_cache: &verified_cache, cross_invariants_source: bundle.cross_invariants_toml.as_deref(), verification_mode: bootstrap::BootstrapSpecVerificationMode::TrustBundle, + key_contract_activation_epochs: Some( + &key_contract_cutover + .as_ref() + .expect("spec reconcile prepared key-contract cutover") + .activation_epochs, + ), }, ); @@ -1359,6 +1440,16 @@ pub(super) async fn install_os_app_with_plan( // newly bootstrapped trigger graph is active for subsequent traffic. state.server.rebuild_reaction_dispatcher(); } + if let Some(key_contract_cutover) = key_contract_cutover.as_mut() { + state + .server + .finish_key_index_contract_activation( + publication_guard, + &tenant_id, + key_contract_cutover, + ) + .await?; + } // App installs can add or change cross-entity reactions. Refresh the live // dispatcher immediately so the newly registered tenant config takes effect @@ -1369,174 +1460,47 @@ pub(super) async fn install_os_app_with_plan( // ── Step 3: Load Cedar policies into memory. ──────────────────── if let Some(ref policy_text) = combined_policy { - if let Err(e) = state + state .server .authz .reload_tenant_policies(tenant, policy_text) - { - tracing::warn!( - tenant, - error = %e, - "Failed to reload tenant Cedar policies after os-app install" - ); - } else { - let mut policies = state.server.tenant_policies.write().unwrap(); // ci-ok: infallible lock - policies.insert(tenant.to_string(), policy_text.clone()); - } - } - - // ── Step 4: Persist/register WASM modules, warming only eager modules. ── - // - // Source-aware preservation: if a (tenant, module) row in the durable store - // has source='upload' (i.e., a hot upload from `POST /api/wasm/modules/{name}`), - // preserve uploads newer than the installed app record. Replace uploads when - // the bundle digest changed or durable metadata predates the app record. - let mut wasm_registered = Vec::new(); - let mut wasm_skipped = Vec::new(); - let mut wasm_failures = Vec::new(); - if plan.wasm { - let existing_sources = match state.server.load_wasm_module_sources(tenant).await { - Ok(map) => map, - Err(e) => { - tracing::warn!( - tenant, - error = %e, - "Failed to load existing WASM modules; install pipeline will not honor hot-upload preservation this cycle" - ); - std::collections::BTreeMap::new() - } - }; - - for (module_name, wasm_bytes) in &bundle.wasm_modules { - let module_started = Instant::now(); - let module_config = bundle.wasm_module_configs.get(module_name); - let hash = temper_wasm::WasmEngine::hash_module(wasm_bytes); - let required = module_config.is_some_and(WasmModuleManifest::is_required); - let replace_uploaded_module = - existing_sources.get(module_name).is_some_and(|existing| { - upload_replacement.should_replace(existing) - || (required && existing.source == "upload" && existing.sha256_hash != hash) - }); - if let Some(existing) = existing_sources.get(module_name) - && existing.source == "upload" - && existing.sha256_hash != hash - { - if replace_uploaded_module { - tracing::info!( - tenant, - module = %module_name, - bundled_hash = %hash, - upload_hash = %existing.sha256_hash, - "Replacing stale hot-uploaded WASM module during os-app reconcile" - ); - } else { - tracing::info!( - tenant, - module = %module_name, - bundled_hash = %hash, - upload_hash = %existing.sha256_hash, - "Skipping bundled install: hot-uploaded module preserved" - ); - wasm_skipped.push(module_name.clone()); - continue; - } - } - - let upsert_result = if replace_uploaded_module { - state - .server - .upsert_bundled_wasm_module_replacing_upload( - tenant, - module_name, - wasm_bytes, - &hash, - ) - .await - } else { - state - .server - .upsert_wasm_module(tenant, module_name, wasm_bytes, &hash, "bundled") - .await - }; - if let Err(e) = upsert_result { - tracing::warn!( - tenant, - module = %module_name, - error = %e, - "Failed to persist WASM module to durable store (continuing in-memory only)" - ); - } - { - let mut wasm_reg = state.server.wasm_module_registry.write().unwrap(); // ci-ok: infallible lock - wasm_reg.register(&tenant_id, module_name, &hash); - } - - if matches!( - module_config.map(|config| config.startup_loading), - Some(WasmStartupLoading::Eager) - ) && let Err(e) = state.server.wasm_engine.compile_and_cache(wasm_bytes) - { - wasm_failures.push(module_name.clone()); - tracing::warn!( - tenant, - module = %module_name, - error = %e, - "Failed to eagerly compile WASM module from OS app" - ); - } - - tracing::info!( - tenant, - module = %module_name, - hash = %hash, - size = wasm_bytes.len(), - duration_ms = module_started.elapsed().as_millis() as u64, - startup_loading = ?module_config - .map(|config| config.startup_loading) - .unwrap_or_default(), - "WASM module registered from OS app" - ); - wasm_registered.push(module_name.clone()); - } - - for (module_name, module_config) in &bundle.wasm_module_configs { - if bundle.wasm_modules.contains_key(module_name) { - continue; - } - match module_config.criticality { - WasmModuleCriticality::Optional => { - wasm_skipped.push(module_name.clone()); - tracing::warn!( - tenant, - module = %module_name, - "Configured optional WASM module artifact is missing from the app bundle" - ); - } - WasmModuleCriticality::PlatformRequired | WasmModuleCriticality::AppRequired => { - wasm_failures.push(module_name.clone()); - tracing::error!( - tenant, - module = %module_name, - criticality = ?module_config.criticality, - "Configured required WASM module artifact is missing from the app bundle" - ); - } - } - } + .map_err(|error| { + format!("Failed to activate tenant Cedar policies for os-app '{app_name}': {error}") + })?; + let mut policies = state + .server + .tenant_policies + .write() + .expect("tenant policy lock poisoned"); + policies.insert(tenant.to_string(), policy_text.clone()); } - tracing::info!( - "Installed os-app '{app_name}' for tenant '{tenant}': added={added:?} updated={updated:?} skipped={skipped:?} wasm={wasm_registered:?}" - ); - - // ARN-68: re-key changed-key-set types after registration (boot-race fix; see the helper + ADR-0153). - if plan.specs && (!added.is_empty() || !updated.is_empty()) { - reconcile::spawn_key_index_rekey_after_spec_change(state, &tenant_id); - } + // ── Step 4: Activate the durable WASM generation in memory. ────────── + let wasm_activation = activate_wasm_generation( + state, + tenant, + app_name, + plan, + &bundle, + &tenant_id, + &effective_wasm_hashes, + &existing_sources, + &upload_replacement, + ) + .await?; + let wasm_registered = wasm_activation.registered; + let wasm_skipped = wasm_activation.skipped; + let wasm_failures = wasm_activation.failures; + let publication_agent_ctx = state.server.spec_publication_dispatch_context( + publication_guard, + &tenant_id, + "platform-bootstrap", + )?; // ── Step 5: Bootstrap App entity + APP.md. ────────────────────────── let (agents_bootstrapped, skills_bootstrapped, adrs_bootstrapped) = if plan.content { - let app_id = bootstrap_app_entity(state, &tenant_id, tenant, app_name).await; + let app_id = + bootstrap_app_entity(state, &tenant_id, tenant, app_name, &publication_agent_ctx).await; // ── Step 6: Bootstrap agents (returns name→uuid map). ────────── let (agents_bootstrapped, agent_uuid_map) = agent_bootstrap::bootstrap_agents( @@ -1545,19 +1509,86 @@ pub(super) async fn install_os_app_with_plan( tenant, &bundle.agents, app_id.as_deref(), + &publication_agent_ctx, ) .await; // ── Step 7: Bootstrap skills (agent-scoped + system). ────────── - let skills_bootstrapped = - bootstrap_skills(state, &tenant_id, tenant, &bundle.skills, &agent_uuid_map).await; + let skills_bootstrapped = bootstrap_skills( + state, + &tenant_id, + tenant, + &bundle.skills, + &agent_uuid_map, + &publication_agent_ctx, + ) + .await; // ── Step 7b: Bootstrap system files (e.g. mode-instructions). ─ - system_files::bootstrap_system_files(state, &tenant_id, tenant, &bundle.system_files).await; + let system_files_bootstrapped = system_files::bootstrap_system_files( + state, + &tenant_id, + tenant, + &bundle.system_files, + &publication_agent_ctx, + ) + .await; // ── Step 8: Bootstrap ADRs into TemperFS. ───────────────────── - let adrs_bootstrapped = - bootstrap_adrs(state, &tenant_id, tenant, app_name, &bundle.adrs).await; + let adrs_bootstrapped = bootstrap_adrs( + state, + &tenant_id, + tenant, + app_name, + &bundle.adrs, + &publication_agent_ctx, + ) + .await?; + + let app_entity_required = state + .registry + .read() + .expect("OS app registry lock poisoned") + .get_spec(&tenant_id, "App") + .is_some(); + let mut incomplete = Vec::new(); + if app_entity_required && app_id.is_none() { + incomplete.push("App entity".to_string()); + } + if agents_bootstrapped.len() != bundle.agents.len() { + incomplete.push(format!( + "agents {}/{}", + agents_bootstrapped.len(), + bundle.agents.len() + )); + } + if skills_bootstrapped.len() != bundle.skills.len() { + incomplete.push(format!( + "skills {}/{}", + skills_bootstrapped.len(), + bundle.skills.len() + )); + } + if system_files_bootstrapped.len() != bundle.system_files.len() { + incomplete.push(format!( + "system files {}/{}", + system_files_bootstrapped.len(), + bundle.system_files.len() + )); + } + if adrs_bootstrapped.len() != bundle.adrs.len() { + incomplete.push(format!( + "ADRs {}/{}", + adrs_bootstrapped.len(), + bundle.adrs.len() + )); + } + if !incomplete.is_empty() { + return Err(format!( + "OS app '{app_name}' content bootstrap is incomplete: {}", + incomplete.join(", ") + )); + } (agents_bootstrapped, skills_bootstrapped, adrs_bootstrapped) } else { @@ -1566,12 +1597,65 @@ pub(super) async fn install_os_app_with_plan( // ── Step 9: Create seed instances. ─────────────────────────────── let seed_created = if plan.seed { - bootstrap_seed_data(state, &tenant_id, tenant, &bundle.seed_instances).await + let created = bootstrap_seed_data( + state, + &tenant_id, + tenant, + &bundle.seed_instances, + &publication_agent_ctx, + ) + .await; + if created.len() != bundle.seed_instances.len() { + return Err(format!( + "OS app '{app_name}' seed bootstrap is incomplete: {}/{} instances ready", + created.len(), + bundle.seed_instances.len() + )); + } + created } else { Vec::new() }; - reconcile::record_app_install_metadata_for_bundle(state, tenant, app_name, &bundle).await; + let platform_store = state + .server + .storage_stack + .as_ref() + .and_then(|stack| stack.platform.clone()); + let mut installed_record = publication_record; + installed_record.status = "installed".to_string(); + let terminal_metadata_write = async move { + if let Some(store) = platform_store { + store + .record_installed_app_metadata(&installed_record) + .await + .map_err(|error| { + format!( + "OS app '{app_name}' completed runtime/content work but failed to finalize durable install metadata: {error}" + ) + })?; + } + Ok(()) + }; + finalize_os_app_publication( + state, + publication_guard, + &tenant_id, + || { + // ARN-68: schedule changed-key-set and vector projection repair + // after the complete runtime/content/seed cutover but before the + // cancellable terminal metadata acknowledgement. + if plan.specs && (!added.is_empty() || !updated.is_empty()) { + reconcile::spawn_key_index_rekey_after_spec_change(state, &tenant_id); + } + }, + terminal_metadata_write, + ) + .await?; + + tracing::info!( + "Installed os-app '{app_name}' for tenant '{tenant}': added={added:?} updated={updated:?} skipped={skipped:?} wasm={wasm_registered:?}" + ); Ok(InstallResult { added, @@ -1587,80 +1671,6 @@ pub(super) async fn install_os_app_with_plan( }) } -#[derive(Debug, Clone, Default)] -struct UploadedWasmReplacementContext { - bundle_wasm_digest_changed: bool, - app_recorded_at: Option>, -} - -impl UploadedWasmReplacementContext { - fn should_replace(&self, existing: &WasmModuleSource) -> bool { - if existing.source != "upload" { - return false; - } - if self.bundle_wasm_digest_changed { - return true; - } - let Some(app_recorded_at) = self.app_recorded_at else { - return false; - }; - let Some(updated_at) = existing - .updated_at - .as_deref() - .and_then(parse_persisted_timestamp) - else { - return false; - }; - updated_at < app_recorded_at - } -} - -fn parse_persisted_timestamp(value: &str) -> Option> { - DateTime::parse_from_rfc3339(value) - .map(|dt| dt.with_timezone(&Utc)) - .or_else(|_| { - NaiveDateTime::parse_from_str(value, "%Y-%m-%d %H:%M:%S").map(|dt| dt.and_utc()) - }) - .ok() -} - -async fn uploaded_wasm_replacement_context( - state: &PlatformState, - tenant: &str, - app_name: &str, - bundle: &AppBundle, -) -> UploadedWasmReplacementContext { - let Some(ps) = state - .server - .storage_stack - .as_ref() - .and_then(|stack| stack.platform.clone()) - else { - return UploadedWasmReplacementContext::default(); - }; - let digest = reconcile::digest_app_bundle(app_name, bundle); - match ps.get_installed_app(tenant, app_name).await { - Ok(Some(record)) => UploadedWasmReplacementContext { - bundle_wasm_digest_changed: record.wasm_digest != digest.wasm_digest, - app_recorded_at: record - .last_reconciled_at - .as_deref() - .or(record.installed_at.as_deref()) - .and_then(parse_persisted_timestamp), - }, - Ok(None) => UploadedWasmReplacementContext::default(), - Err(error) => { - tracing::warn!( - tenant, - app = %app_name, - error = %error, - "Failed to read OS app metadata while deciding WASM hot-upload replacement" - ); - UploadedWasmReplacementContext::default() - } - } -} - // ── Bootstrap helpers (entity creation during install) ─────────────── /// Bootstrap an App entity for the installed OS app. @@ -1672,6 +1682,7 @@ async fn bootstrap_app_entity( tenant_id: &TenantId, tenant: &str, app_name: &str, + agent_ctx: &temper_server::request_context::AgentContext, ) -> Option { // Check if App entity type is registered. let has_apps = { @@ -1687,15 +1698,13 @@ async fn bootstrap_app_entity( return None; } - let agent_ctx = temper_server::request_context::AgentContext::for_service("platform-bootstrap"); - // Look for an existing App entity with this name. let existing_ids = state.server.list_entity_ids_lazy(tenant_id, "App").await; let mut existing_app_id = None; for id in &existing_ids { if let Ok(resp) = state .server - .get_tenant_entity_state(tenant_id, "App", id) + .get_tenant_entity_state_in_generation(tenant_id, "App", id, agent_ctx) .await && let Some(name) = state_field_str(&resp.state.fields, &["Name", "name"]) && name.eq_ignore_ascii_case(app_name) @@ -1730,7 +1739,7 @@ async fn bootstrap_app_entity( && registry.get_spec(tenant_id, "Workspace").is_some() }; if has_fs { - if let Err(e) = ensure_app_docs_workspace(state, tenant_id, &agent_ctx).await { + if let Err(e) = ensure_app_docs_workspace(state, tenant_id, agent_ctx).await { tracing::warn!(tenant, app = %app_name, error = %e, "Failed to ensure app docs workspace"); } else { let app_slug = slug_fragment(app_name); @@ -1739,7 +1748,7 @@ async fn bootstrap_app_entity( if let Err(e) = ensure_directory( state, tenant_id, - &agent_ctx, + agent_ctx, DirectoryBootstrapTarget { directory_id: &app_dir_id, name: app_name, @@ -1757,7 +1766,7 @@ async fn bootstrap_app_entity( if let Err(e) = ensure_markdown_file( state, tenant_id, - &agent_ctx, + agent_ctx, MarkdownFileBootstrapTarget { file_id: &file_id_str, name: "APP.md", @@ -1794,7 +1803,7 @@ async fn bootstrap_app_entity( "version": version, "app_guide_file_id": app_guide_file_id, }), - agent_ctx: &agent_ctx, + agent_ctx, await_integration: false, await_reactions: true, }) @@ -1807,7 +1816,13 @@ async fn bootstrap_app_entity( let new_app_id = format!("ap-{}", temper_runtime::scheduler::sim_uuid()); match state .server - .get_or_create_tenant_entity(tenant_id, "App", &new_app_id, serde_json::json!({})) + .get_or_create_tenant_entity_in_generation( + tenant_id, + "App", + &new_app_id, + serde_json::json!({}), + agent_ctx, + ) .await { Ok(_) => { @@ -1826,7 +1841,7 @@ async fn bootstrap_app_entity( "version": version, "app_guide_file_id": app_guide_file_id, }), - agent_ctx: &agent_ctx, + agent_ctx, await_integration: false, await_reactions: true, }) @@ -1869,6 +1884,7 @@ async fn bootstrap_skills( tenant: &str, skills: &[AppSkillDefinition], agent_uuid_map: &BTreeMap, + agent_ctx: &temper_server::request_context::AgentContext, ) -> Vec { if skills.is_empty() { return Vec::new(); @@ -1889,10 +1905,8 @@ async fn bootstrap_skills( return Vec::new(); } - let agent_ctx = temper_server::request_context::AgentContext::for_service("platform-bootstrap"); - // Ensure workspace exists. - if let Err(e) = ensure_app_docs_workspace(state, tenant_id, &agent_ctx).await { + if let Err(e) = ensure_app_docs_workspace(state, tenant_id, agent_ctx).await { tracing::warn!(tenant, error = %e, "Failed to ensure app docs workspace for skill bootstrap"); return Vec::new(); } @@ -1901,7 +1915,7 @@ async fn bootstrap_skills( if let Err(e) = ensure_directory( state, tenant_id, - &agent_ctx, + agent_ctx, DirectoryBootstrapTarget { directory_id: "os-system-root", name: "system", @@ -1920,7 +1934,7 @@ async fn bootstrap_skills( if let Err(e) = ensure_directory( state, tenant_id, - &agent_ctx, + agent_ctx, DirectoryBootstrapTarget { directory_id: "os-system-skills-root", name: "skills", @@ -1939,7 +1953,7 @@ async fn bootstrap_skills( if let Err(e) = ensure_directory( state, tenant_id, - &agent_ctx, + agent_ctx, DirectoryBootstrapTarget { directory_id: "os-agents-root", name: "agents", @@ -1996,7 +2010,7 @@ async fn bootstrap_skills( if let Err(e) = ensure_directory( state, tenant_id, - &agent_ctx, + agent_ctx, DirectoryBootstrapTarget { directory_id: &agent_dir_id, name: agent_name, @@ -2018,7 +2032,7 @@ async fn bootstrap_skills( if let Err(e) = ensure_directory( state, tenant_id, - &agent_ctx, + agent_ctx, DirectoryBootstrapTarget { directory_id: &agent_skills_dir_id, name: "skills", @@ -2045,7 +2059,7 @@ async fn bootstrap_skills( if let Err(e) = ensure_directory( state, tenant_id, - &agent_ctx, + agent_ctx, DirectoryBootstrapTarget { directory_id: &dir_id, name: &slug, @@ -2064,7 +2078,7 @@ async fn bootstrap_skills( if let Err(e) = ensure_markdown_file( state, tenant_id, - &agent_ctx, + agent_ctx, MarkdownFileBootstrapTarget { file_id: &file_id, name: "SKILL.md", @@ -2092,7 +2106,7 @@ async fn bootstrap_skills( if let Err(e) = ensure_markdown_file( state, tenant_id, - &agent_ctx, + agent_ctx, MarkdownFileBootstrapTarget { file_id: &comp_file_id, name: comp_file_name, @@ -2142,9 +2156,10 @@ async fn bootstrap_adrs( tenant: &str, app_name: &str, adrs: &[AdrEntry], -) -> Vec { + agent_ctx: &temper_server::request_context::AgentContext, +) -> Result, String> { if adrs.is_empty() { - return Vec::new(); + return Ok(Vec::new()); } let has_workspace = { @@ -2166,18 +2181,21 @@ async fn bootstrap_adrs( count = adrs.len(), "Skipping ADR bootstrap — TemperFS types not registered (install temper-fs first)" ); - return Vec::new(); + return Err(format!( + "ADR bootstrap for OS app '{app_name}' requires Workspace, Directory, and File" + )); } - let agent_ctx = temper_server::request_context::AgentContext::for_service("platform-bootstrap"); - if let Err(error) = ensure_app_docs_workspace(state, tenant_id, &agent_ctx).await { + if let Err(error) = ensure_app_docs_workspace(state, tenant_id, agent_ctx).await { tracing::warn!( tenant, app = %app_name, error = %error, "Failed to ensure app docs workspace for ADR bootstrap" ); - return Vec::new(); + return Err(format!( + "failed to ensure app docs workspace for ADR bootstrap: {error}" + )); } let app_slug = slug_fragment(app_name); @@ -2186,7 +2204,7 @@ async fn bootstrap_adrs( if let Err(error) = ensure_directory( state, tenant_id, - &agent_ctx, + agent_ctx, DirectoryBootstrapTarget { directory_id: &app_dir_id, name: app_name, @@ -2203,7 +2221,9 @@ async fn bootstrap_adrs( error = %error, "Failed to ensure app ADR directory" ); - return Vec::new(); + return Err(format!( + "failed to ensure app ADR directory '{app_dir_path}': {error}" + )); } let adrs_dir_id = format!("os-app-docs-dir-{app_slug}-adrs"); @@ -2211,7 +2231,7 @@ async fn bootstrap_adrs( if let Err(error) = ensure_directory( state, tenant_id, - &agent_ctx, + agent_ctx, DirectoryBootstrapTarget { directory_id: &adrs_dir_id, name: "adrs", @@ -2228,7 +2248,9 @@ async fn bootstrap_adrs( error = %error, "Failed to ensure app ADR subdirectory" ); - return Vec::new(); + return Err(format!( + "failed to ensure app ADR directory '{adrs_dir_path}': {error}" + )); } let mut bootstrapped = Vec::new(); @@ -2236,10 +2258,10 @@ async fn bootstrap_adrs( let file_slug = slug_fragment(&adr.name); let file_id = format!("os-app-adr-{app_slug}-{file_slug}"); let file_path = format!("{adrs_dir_path}/{}", adr.file_name); - match ensure_markdown_file( + ensure_markdown_file( state, tenant_id, - &agent_ctx, + agent_ctx, MarkdownFileBootstrapTarget { file_id: &file_id, name: &adr.file_name, @@ -2250,21 +2272,16 @@ async fn bootstrap_adrs( adr.content.as_bytes(), ) .await - { - Ok(()) => bootstrapped.push(file_path), - Err(error) => { - tracing::warn!( - tenant, - app = %app_name, - adr = %adr.file_name, - error = %error, - "Failed to bootstrap ADR into TemperFS" - ); - } - } + .map_err(|error| { + format!( + "failed to bootstrap ADR '{}' into TemperFS: {error}", + adr.file_name + ) + })?; + bootstrapped.push(file_path); } - bootstrapped + Ok(bootstrapped) } pub(super) async fn ensure_app_docs_workspace( @@ -2274,16 +2291,22 @@ pub(super) async fn ensure_app_docs_workspace( ) -> Result<(), String> { if !state .server - .ensure_entity_loaded(tenant_id, "Workspace", APP_DOCS_WORKSPACE_ID) + .ensure_entity_loaded_in_generation( + tenant_id, + "Workspace", + APP_DOCS_WORKSPACE_ID, + agent_ctx, + ) .await { state .server - .get_or_create_tenant_entity( + .get_or_create_tenant_entity_in_generation( tenant_id, "Workspace", APP_DOCS_WORKSPACE_ID, serde_json::json!({}), + agent_ctx, ) .await .map_err(|e| format!("failed to create app docs workspace entity: {e}"))?; @@ -2337,7 +2360,7 @@ pub(super) async fn ensure_directory( ) -> Result<(), String> { if state .server - .ensure_entity_loaded(tenant_id, "Directory", target.directory_id) + .ensure_entity_loaded_in_generation(tenant_id, "Directory", target.directory_id, agent_ctx) .await { ensure_entity_field_aliases( @@ -2351,6 +2374,7 @@ pub(super) async fn ensure_directory( "ParentId": target.parent_id, "WorkspaceId": target.workspace_id, }), + agent_ctx, ) .await?; return Ok(()); @@ -2358,11 +2382,12 @@ pub(super) async fn ensure_directory( state .server - .get_or_create_tenant_entity( + .get_or_create_tenant_entity_in_generation( tenant_id, "Directory", target.directory_id, serde_json::json!({}), + agent_ctx, ) .await .map_err(|e| { @@ -2461,7 +2486,7 @@ pub(super) async fn ensure_markdown_file( }); let existed = state .server - .ensure_entity_loaded(tenant_id, "File", target.file_id) + .ensure_entity_loaded_in_generation(tenant_id, "File", target.file_id, agent_ctx) .await; if !existed { state @@ -2511,10 +2536,18 @@ pub(super) async fn ensure_markdown_file( create_fields.clone(), ) .await?; - ensure_entity_field_aliases(state, tenant_id, "File", target.file_id, canonical_fields).await?; + ensure_entity_field_aliases( + state, + tenant_id, + "File", + target.file_id, + canonical_fields, + agent_ctx, + ) + .await?; let desired_hash = content_sha256(content); - if file_already_contains(state, tenant_id, target.file_id, &desired_hash).await? { + if file_already_contains(state, tenant_id, target.file_id, &desired_hash, agent_ctx).await? { return Ok(()); } @@ -2546,12 +2579,12 @@ async fn bootstrap_seed_data( tenant_id: &TenantId, tenant: &str, instances: &[SeedInstance], + agent_ctx: &temper_server::request_context::AgentContext, ) -> Vec { if instances.is_empty() { return Vec::new(); } - let agent_ctx = temper_server::request_context::AgentContext::for_service("platform-bootstrap"); let mut created = Vec::new(); for instance in instances { @@ -2581,19 +2614,18 @@ async fn bootstrap_seed_data( ) }); - // Check if entity already exists (idempotent). - if state + // Create the entity once, then replay the declared seed actions with + // stable idempotency keys so an interrupted install can resume. + let exists = state .server - .entity_exists(tenant_id, &instance.entity_type, &entity_id) - { + .entity_exists(tenant_id, &instance.entity_type, &entity_id); + if exists { tracing::debug!( tenant, entity_type = %instance.entity_type, entity_id = %entity_id, - "Seed entity already exists — skipping" + "Seed entity already exists — resuming declared actions" ); - created.push(format!("{}({})", instance.entity_type, entity_id)); - continue; } // Create entity with initial fields. @@ -2603,66 +2635,90 @@ async fn bootstrap_seed_data( instance.fields.clone() }; - match state - .server - .get_or_create_tenant_entity( - tenant_id, - &instance.entity_type, - &entity_id, - initial_fields, - ) - .await + if !exists + && let Err(e) = state + .server + .get_or_create_tenant_entity_in_generation( + tenant_id, + &instance.entity_type, + &entity_id, + initial_fields, + agent_ctx, + ) + .await { - Ok(_) => { - // Dispatch each action in order. - for action in &instance.actions { - let params = if action.params.is_null() { - serde_json::json!({}) - } else { - action.params.clone() - }; - if let Err(e) = state - .server - .dispatch(temper_server::state::DispatchCommand { - tenant: tenant_id, - entity_type: &instance.entity_type, - entity_id: &entity_id, - action: &action.name, - params, - agent_ctx: &agent_ctx, - await_integration: false, - await_reactions: true, - }) - .await - { - tracing::warn!( - tenant, - entity_type = %instance.entity_type, - entity_id = %entity_id, - action = %action.name, - error = %e, - "Failed to dispatch seed action" - ); - } + tracing::warn!( + tenant, + entity_type = %instance.entity_type, + entity_id = %entity_id, + error = %e, + "Failed to create seed entity" + ); + continue; + } + + let mut actions_ready = true; + for (action_index, action) in instance.actions.iter().enumerate() { + let params = if action.params.is_null() { + serde_json::json!({}) + } else { + action.params.clone() + }; + let mut action_ctx = agent_ctx.clone(); + action_ctx.idempotency_key = Some(format!( + "os-seed:{}:{}:{action_index}:{}", + instance.entity_type, entity_id, action.name + )); + match state + .server + .dispatch(temper_server::state::DispatchCommand { + tenant: tenant_id, + entity_type: &instance.entity_type, + entity_id: &entity_id, + action: &action.name, + params, + agent_ctx: &action_ctx, + await_integration: false, + await_reactions: true, + }) + .await + { + Ok(response) if response.success => {} + Ok(response) => { + actions_ready = false; + tracing::warn!( + tenant, + entity_type = %instance.entity_type, + entity_id = %entity_id, + action = %action.name, + error = response.error.as_deref().unwrap_or("seed action rejected"), + "Failed to dispatch seed action" + ); + break; + } + Err(e) => { + actions_ready = false; + tracing::warn!( + tenant, + entity_type = %instance.entity_type, + entity_id = %entity_id, + action = %action.name, + error = %e, + "Failed to dispatch seed action" + ); + break; } - tracing::info!( - tenant, - entity_type = %instance.entity_type, - entity_id = %entity_id, - "Seed entity created" - ); - created.push(format!("{}({})", instance.entity_type, entity_id)); - } - Err(e) => { - tracing::warn!( - tenant, - entity_type = %instance.entity_type, - entity_id = %entity_id, - error = %e, - "Failed to create seed entity" - ); } } + if actions_ready { + tracing::info!( + tenant, + entity_type = %instance.entity_type, + entity_id = %entity_id, + "Seed entity ready" + ); + created.push(format!("{}({})", instance.entity_type, entity_id)); + } } created } diff --git a/crates/temper-platform/src/os_apps/mod_test.rs b/crates/temper-platform/src/os_apps/mod_test.rs index 42f887e25..32c8e2457 100644 --- a/crates/temper-platform/src/os_apps/mod_test.rs +++ b/crates/temper-platform/src/os_apps/mod_test.rs @@ -2,6 +2,63 @@ use super::agent_bootstrap::{ AgentSoulRefreshDecision, bootstrapped_agent_soul_entity_id, decide_agent_soul_refresh, }; use super::*; + +#[test] +fn os_app_publication_intent_binds_every_persisted_provenance_field() { + let baseline = InstalledAppRecord { + tenant: "tenant-a".to_string(), + app_name: "project-management".to_string(), + source_kind: "genesis".to_string(), + app_ref: "temper/project-management@abc".to_string(), + version_hash: "abc".to_string(), + pinned_version_hash: "abc".to_string(), + current_version_hash: "abc".to_string(), + follow_policy: "pinned".to_string(), + closure_id: "closure-a".to_string(), + registry_url: "https://registry.example".to_string(), + registry_tenant: "source-a".to_string(), + app_version: "1.0.0".to_string(), + bundle_digest: "bundle".to_string(), + spec_digest: "spec".to_string(), + policy_digest: "policy".to_string(), + wasm_digest: "wasm".to_string(), + content_digest: "content".to_string(), + seed_digest: "seed".to_string(), + installed_at: None, + last_reconciled_at: None, + status: "installed".to_string(), + }; + let intent = |record: &InstalledAppRecord| { + let mut components = Vec::new(); + append_installed_app_record_intent(&mut components, record); + temper_server::ServerState::spec_publication_intent( + "os-app-runtime-generation", + components + .iter() + .map(|(name, value)| (name.as_str(), value.as_slice())), + ) + }; + let baseline_intent = intent(&baseline); + for mutate in [ + |record: &mut InstalledAppRecord| record.source_kind.push_str("-other"), + |record: &mut InstalledAppRecord| record.app_ref.push_str("-other"), + |record: &mut InstalledAppRecord| record.version_hash.push_str("-other"), + |record: &mut InstalledAppRecord| record.pinned_version_hash.push_str("-other"), + |record: &mut InstalledAppRecord| record.current_version_hash.push_str("-other"), + |record: &mut InstalledAppRecord| record.follow_policy.push_str("-other"), + |record: &mut InstalledAppRecord| record.closure_id.push_str("-other"), + |record: &mut InstalledAppRecord| record.registry_url.push_str("-other"), + |record: &mut InstalledAppRecord| record.registry_tenant.push_str("-other"), + ] { + let mut changed = baseline.clone(); + mutate(&mut changed); + assert_ne!( + intent(&changed), + baseline_intent, + "persisted provenance must change the sticky publication intent" + ); + } +} use std::collections::HashMap; use std::fs; use std::time::Duration; @@ -267,6 +324,40 @@ fn test_reconcile_plan_for_wasm_only_digest_skips_unrelated_phases() { ); } +#[test] +fn publishing_record_retries_every_unproved_post_commit_phase() { + let digest = OsAppBundleDigest { + app_name: "publication-retry".to_string(), + app_version: "1.0.0".to_string(), + bundle_digest: "sha256:bundle".to_string(), + spec_digest: "sha256:spec".to_string(), + policy_digest: "sha256:policy".to_string(), + wasm_digest: "sha256:wasm".to_string(), + content_digest: "sha256:content".to_string(), + seed_digest: "sha256:seed".to_string(), + }; + let record = InstalledAppRecord { + tenant: "default".to_string(), + app_name: digest.app_name.clone(), + app_version: digest.app_version.clone(), + bundle_digest: digest.bundle_digest.clone(), + spec_digest: digest.spec_digest.clone(), + policy_digest: digest.policy_digest.clone(), + wasm_digest: digest.wasm_digest.clone(), + content_digest: digest.content_digest.clone(), + seed_digest: digest.seed_digest.clone(), + status: "publishing:{\"bundle_wasm_digest_changed\":false,\"app_recorded_at\":null}" + .to_string(), + ..InstalledAppRecord::default() + }; + + assert_eq!( + reconcile::plan_reconcile_from_installed_record(&record, &digest, true, true, true), + OsAppInstallPlan::all(), + "target digests in an in-progress record do not prove runtime, content, or seed completion" + ); +} + #[tokio::test] async fn test_reconcile_os_app_skips_unchanged_bundle_digest() { let db_path = format!("/tmp/temper-test-digest-{}.db", uuid::Uuid::new_v4()); @@ -541,6 +632,72 @@ async fn test_runtime_recovery_requires_active_policies_for_ready_outcome() { let _ = std::fs::remove_file(format!("{db_path}-shm")); } +#[tokio::test] +async fn test_runtime_recovery_reconciles_a_warm_but_incomplete_publication() { + let db_path = format!( + "/tmp/temper-test-runtime-publishing-{}.db", + uuid::Uuid::new_v4() + ); + let db_url = format!("file:{db_path}"); + let tenant = "test-runtime-publishing"; + let app_name = "project-management"; + + let turso = temper_store_turso::TursoEventStore::new(&db_url, None) + .await + .unwrap(); + let mut state = PlatformState::new(None); + state + .server + .set_storage_stack(temper_server::StorageStack::from_turso(turso)); + install_os_app(&state, tenant, app_name) + .await + .expect("initial install should succeed"); + + let ps = state + .server + .storage_stack + .as_ref() + .and_then(|stack| stack.platform.clone()) + .expect("platform store"); + let mut record = ps + .get_installed_app(tenant, app_name) + .await + .expect("read installed metadata") + .expect("installed metadata"); + record.status = + "publishing:{\"bundle_wasm_digest_changed\":false,\"app_recorded_at\":null}".to_string(); + ps.record_installed_app_metadata(&record) + .await + .expect("simulate crash after runtime publication but before finalization"); + + assert_eq!( + crate::recovery::recover_installed_app_runtime_state( + &state, + ps.as_ref(), + tenant, + app_name, + ) + .await, + crate::recovery::InstalledAppRuntimeRecoveryOutcome::NeedsReconcile, + "warm runtime artifacts cannot prove post-publication content and seed completion" + ); + + crate::recovery::restore_installed_apps(&state, ps.as_ref()).await; + let recovered = ps + .get_installed_app(tenant, app_name) + .await + .expect("read recovered metadata") + .expect("recovered metadata"); + assert_eq!( + recovered.status, "installed", + "startup recovery must run the full idempotent reconcile and finalize metadata" + ); + + let _ = std::fs::remove_file(&db_path); + let _ = std::fs::remove_file(format!("{db_path}-wal")); + let _ = std::fs::remove_file(format!("{db_path}-shm")); +} + #[tokio::test] async fn test_install_plan_without_spec_phase_does_not_reclassify_specs() { let state = PlatformState::new(None); @@ -821,11 +978,11 @@ async fn test_reconcile_os_app_repairs_entity_set_map_from_matching_digest() { let result = reconcile_os_app(&state, tenant_name, "project-management") .await - .expect("reconcile should heal missing entity-set map without reinstalling content"); + .expect("reconcile should heal missing entity-set map in one guarded generation"); assert!( - matches!(result, OsAppReconcileResult::Skipped { .. }), - "matching digest should repair OData entity-set mappings without reinstall, got {result:?}" + matches!(result, OsAppReconcileResult::Installed { .. }), + "runtime mapping drift requires one guarded idempotent reinstall, got {result:?}" ); assert_eq!( state @@ -1372,7 +1529,7 @@ async fn test_install_os_app_persists_granular_policy_rows() { .expect("load granular policies"); assert!( rows.iter().any(|row| { - row.policy_id == "project-management-issue" + row.policy_id == os_app_policy_row_id("project-management", "policies/issue.cedar") && row.enabled && row.cedar_text.contains("resource is Issue") }), @@ -1387,6 +1544,162 @@ async fn test_install_os_app_persists_granular_policy_rows() { .any(|(tenant, text)| tenant == "test-policy-rows" && text.contains("Issue")), "legacy aggregate policy should still be persisted for compatibility" ); + + temper_server::authz::load_and_activate_tenant_policies(&state.server, "test-policy-rows") + .await; + let reloaded_rows = policy_store + .load_policies_for_tenant("test-policy-rows") + .await + .expect("reload granular policies after compatibility projection"); + assert!( + reloaded_rows.iter().all(|row| row.policy_id != "primary"), + "the compatibility aggregate must not be copied back into a primary row: {reloaded_rows:?}" + ); +} + +#[tokio::test] +async fn test_subsumed_multi_owner_policy_reload_does_not_create_ownerless_primary() { + use temper_store_turso::TursoEventStore; + + let tenant = "test-policy-owner-reload"; + let shared = r#"permit(principal, action == Action::\"shared\", resource);"#; + let owner_a = + format!("{shared}\npermit(principal, action == Action::\"owner_a_only\", resource);"); + let aggregate = merge_bundle_policies("", &[owner_a.clone(), shared.to_string()]); + assert_eq!( + aggregate, owner_a, + "the compatibility aggregate should deduplicate the subsumed owner row" + ); + + let db_path = format!( + "/tmp/temper-policy-owner-reload-{}.db", + uuid::Uuid::new_v4() + ); + let db_url = format!("file:{db_path}"); + let turso = TursoEventStore::new(&db_url, None).await.unwrap(); + let owner_a_entries = [("owner-a", owner_a.as_str(), "os-app:owner-a")]; + turso + .publish_specs( + tenant, + &[], + false, + None, + Some(&aggregate), + None, + &[], + Some("os-app:owner-a"), + &owner_a_entries, + ) + .await + .expect("publish first policy owner"); + let owner_b_entries = [("owner-b", shared, "os-app:owner-b")]; + turso + .publish_specs( + tenant, + &[], + false, + None, + Some(&aggregate), + None, + &[], + Some("os-app:owner-b"), + &owner_b_entries, + ) + .await + .expect("publish subsumed second policy owner"); + + let mut state = PlatformState::new(None); + state + .server + .set_storage_stack(temper_server::StorageStack::from_turso(turso.clone())); + temper_server::authz::load_and_activate_tenant_policies(&state.server, tenant).await; + + turso + .publish_specs( + tenant, + &[], + false, + None, + Some(shared), + None, + &[], + Some("os-app:owner-a"), + &[], + ) + .await + .expect("remove first policy owner's complete row set"); + temper_server::authz::load_and_activate_tenant_policies(&state.server, tenant).await; + + let rows = state + .server + .policy_store() + .expect("policy store") + .load_policies_for_tenant(tenant) + .await + .expect("reload granular policy owners"); + assert_eq!(rows.len(), 1, "only the surviving owner row may remain"); + assert_eq!(rows[0].policy_id, "owner-b"); + assert_eq!(rows[0].created_by, "os-app:owner-b"); + assert!( + rows.iter().all(|row| row.policy_id != "primary"), + "a deduplicated compatibility aggregate must never become ownerless authority: {rows:?}" + ); +} + +#[tokio::test] +async fn test_install_os_app_prefers_early_granular_rows_over_legacy_cache() { + use temper_store_turso::TursoEventStore; + + let tenant = "test-policy-primary-migration"; + let legacy = r#"permit(principal, action == Action::"legacy_only", resource);"#; + let early = r#"permit(principal, action == Action::"early_only", resource);"#; + let db_path = format!("/tmp/temper-policy-primary-{}.db", uuid::Uuid::new_v4()); + let db_url = format!("file:{db_path}"); + let turso = TursoEventStore::new(&db_url, None).await.unwrap(); + turso + .upsert_tenant_policy(tenant, legacy) + .await + .expect("persist legacy aggregate"); + turso + .save_policy(tenant, "early", early, "early-writer") + .await + .expect("persist early granular row"); + let mut state = PlatformState::new(None); + state + .server + .authz + .reload_tenant_policies(tenant, legacy) + .expect("activate legacy policy"); + state + .server + .tenant_policies + .write() + .unwrap() + .insert(tenant.to_string(), legacy.to_string()); + state + .server + .set_storage_stack(temper_server::StorageStack::from_turso(turso)); + + install_os_app(&state, tenant, "project-management") + .await + .expect("install must use the canonical granular generation"); + + let rows = state + .server + .policy_store() + .expect("policy store") + .load_policies_for_tenant(tenant) + .await + .expect("load migrated rows"); + assert!(rows.iter().all(|row| row.policy_id != "primary")); + assert!(rows.iter().any(|row| row.policy_id == "early")); + let active = state + .server + .authz + .get_tenant_policy_text(tenant) + .expect("installed generation should be active"); + assert!(!active.contains("legacy_only")); + assert!(active.contains("early_only")); } /// Proves the full install → persist → reboot → restore cycle. @@ -1833,11 +2146,11 @@ mode = "commons" ); assert_eq!( os_app_policy_row_id("katagami-commons", "policies/palette_system.cedar"), - "katagami-commons-palette_system" + "os-app-6b61746167616d692d636f6d6d6f6e73-706f6c69636965732f70616c657474655f73797374656d2e6365646172" ); assert_eq!( os_app_policy_row_id("katagami-commons", "policies/commons/guardrail.cedar"), - "katagami-commons-commons-guardrail" + "os-app-6b61746167616d692d636f6d6d6f6e73-706f6c69636965732f636f6d6d6f6e732f67756172647261696c2e6365646172" ); assert!( bundle @@ -1849,6 +2162,15 @@ mode = "commons" let _ = fs::remove_dir_all(&temp_dir); } +#[test] +fn test_os_app_policy_row_ids_are_disjoint_across_owner_path_boundaries() { + assert_ne!( + os_app_policy_row_id("foo", "policies/bar-baz.cedar"), + os_app_policy_row_id("foo-bar", "policies/baz.cedar"), + "distinct app owners and paths must never share a durable policy row" + ); +} + #[test] fn test_load_app_bundle_carries_wasm_module_contracts() { let temp_dir = diff --git a/crates/temper-platform/src/os_apps/policy_rows.rs b/crates/temper-platform/src/os_apps/policy_rows.rs index aa9a5579c..8f9ccfa9c 100644 --- a/crates/temper-platform/src/os_apps/policy_rows.rs +++ b/crates/temper-platform/src/os_apps/policy_rows.rs @@ -1,58 +1,45 @@ use super::AppBundle; -use crate::state::PlatformState; + +pub(super) struct OwnedPolicyEntry { + pub(super) policy_id: String, + pub(super) cedar_text: String, + pub(super) created_by: String, +} pub(crate) fn os_app_policy_row_id(app_name: &str, relative_path: &str) -> String { - let source = relative_path - .trim_start_matches('/') - .strip_prefix("policies/") - .unwrap_or_else(|| relative_path.trim_start_matches('/')) - .strip_suffix(".cedar") - .unwrap_or(relative_path); - let mut slug = String::new(); - for ch in source.chars() { - if ch.is_ascii_alphanumeric() || ch == '_' || ch == '-' { - slug.push(ch); - } else if ch == '/' || ch == '.' { - slug.push('-'); - } else { - slug.push('_'); + fn hex_component(value: &str) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut encoded = String::with_capacity(value.len() * 2); + for byte in value.as_bytes() { + encoded.push(char::from(HEX[usize::from(byte >> 4)])); + encoded.push(char::from(HEX[usize::from(byte & 0x0f)])); } + encoded } - let slug = slug.trim_matches('-'); - if slug.is_empty() { - format!("{app_name}-policy") - } else { - format!("{app_name}-{slug}") - } + + // Hex encoding is injective and the separator is outside its alphabet, so + // no app owner/path pair can alias another row in the tenant-wide primary + // key. Slug concatenation was ambiguous across both owner and path. + format!( + "os-app-{}-{}", + hex_component(app_name), + hex_component(relative_path.trim_start_matches('/')) + ) } -pub(super) async fn persist_bundle_policy_rows( - state: &PlatformState, - tenant: &str, - app_name: &str, - bundle: &AppBundle, -) -> Result<(), String> { - if bundle.cedar_policy_sources.is_empty() { - return Ok(()); - } - let Some(policy_store) = state.server.policy_store() else { - return Ok(()); - }; +pub(super) fn bundle_policy_entries(app_name: &str, bundle: &AppBundle) -> Vec { let created_by = format!("os-app:{app_name}"); + let mut entries = Vec::new(); for source in &bundle.cedar_policy_sources { let cedar_text = source.text.trim(); if cedar_text.is_empty() { continue; } - let policy_id = os_app_policy_row_id(app_name, &source.relative_path); - policy_store - .save_policy(tenant, &policy_id, cedar_text, &created_by) - .await - .map_err(|error| { - format!( - "Failed to persist OS app Cedar policy row '{policy_id}' for '{app_name}': {error}" - ) - })?; + entries.push(OwnedPolicyEntry { + policy_id: os_app_policy_row_id(app_name, &source.relative_path), + cedar_text: cedar_text.to_string(), + created_by: created_by.clone(), + }); } - Ok(()) + entries } diff --git a/crates/temper-platform/src/os_apps/reconcile.rs b/crates/temper-platform/src/os_apps/reconcile.rs index cc617c804..37b31bd3b 100644 --- a/crates/temper-platform/src/os_apps/reconcile.rs +++ b/crates/temper-platform/src/os_apps/reconcile.rs @@ -6,8 +6,8 @@ use temper_server::platform_store::InstalledAppRecord; use super::{ AppBundle, AppEntry, OsAppBundleDigest, OsAppInstallPlan, OsAppReconcileResult, catalog, - get_os_app, install_os_app_with_plan, os_app_dependencies, - restore_app_specs_from_matching_digest, tenant_has_ready_app_specs_for_bundle, + get_os_app, install_os_app_with_plan_under_guard, os_app_dependencies, + tenant_has_ready_app_specs_for_bundle, }; use crate::state::PlatformState; @@ -236,6 +236,14 @@ pub(super) fn plan_reconcile_from_installed_record( wasm_registered: bool, policies_active: bool, ) -> OsAppInstallPlan { + // The atomic publication transaction records target digests before the + // runtime/WASM/content/seed phases run. A non-installed record therefore + // proves only intent, not completion of any post-commit phase. Retry the + // complete idempotent bundle workflow instead of letting target digests + // suppress work that may never have started. + if record.status != "installed" { + return OsAppInstallPlan::all(); + } OsAppInstallPlan { specs: record.spec_digest != digest.spec_digest || !specs_ready, policies: record.policy_digest != digest.policy_digest || !policies_active, @@ -328,73 +336,46 @@ pub(crate) async fn tenant_has_durable_wasm_for_bundle( }) } -async fn record_app_install_metadata( +/// Reconcile one app without recursively processing dependencies. +/// +/// Callers that need dependencies should first call +/// [`resolve_os_app_install_order`] and reconcile each returned app once. +pub async fn reconcile_os_app( state: &PlatformState, tenant: &str, - digest: &OsAppBundleDigest, - status: &str, -) { - let Some(ps) = state - .server - .storage_stack - .as_ref() - .and_then(|stack| stack.platform.clone()) - else { - return; - }; - - let record = InstalledAppRecord { - tenant: tenant.to_string(), - app_name: digest.app_name.clone(), - source_kind: "local".to_string(), - app_ref: String::new(), - version_hash: String::new(), - pinned_version_hash: String::new(), - current_version_hash: String::new(), - follow_policy: "pinned".to_string(), - closure_id: String::new(), - registry_url: String::new(), - registry_tenant: String::new(), - app_version: digest.app_version.clone(), - bundle_digest: digest.bundle_digest.clone(), - spec_digest: digest.spec_digest.clone(), - policy_digest: digest.policy_digest.clone(), - wasm_digest: digest.wasm_digest.clone(), - content_digest: digest.content_digest.clone(), - seed_digest: digest.seed_digest.clone(), - installed_at: None, - last_reconciled_at: None, - status: status.to_string(), - }; - - if let Err(error) = ps.record_installed_app_metadata(&record).await { - tracing::warn!( - tenant, - app = %digest.app_name, - error = %error, - "Failed to persist OS app digest metadata" - ); - } + app_name: &str, +) -> Result { + reconcile_os_app_after_generation(state, tenant, app_name, None).await } -pub(super) async fn record_app_install_metadata_for_bundle( +/// Reconcile one app after a request-to-publication handoff. When supplied, +/// `expected_generation` rejects an intervening complete tenant generation +/// before any candidate state is read or made durable. +pub(crate) async fn reconcile_os_app_after_generation( state: &PlatformState, tenant: &str, app_name: &str, - bundle: &AppBundle, -) { - let bundle_digest = digest_app_bundle(app_name, bundle); - record_app_install_metadata(state, tenant, &bundle_digest, "installed").await; + expected_generation: Option, +) -> Result { + let tenant_id = TenantId::new(tenant); + let mut publication_guard = match expected_generation { + Some(expected_generation) => { + state + .server + .begin_spec_publication_after_drain(&tenant_id, expected_generation) + .await? + } + None => state.server.begin_spec_publication(&tenant_id).await?, + }; + reconcile_os_app_under_guard(state, tenant, app_name, None, &mut publication_guard).await } -/// Reconcile one app without recursively processing dependencies. -/// -/// Callers that need dependencies should first call -/// [`resolve_os_app_install_order`] and reconcile each returned app once. -pub async fn reconcile_os_app( +pub(crate) async fn reconcile_os_app_under_guard( state: &PlatformState, tenant: &str, app_name: &str, + publication_record_override: Option, + publication_guard: &mut temper_server::state::SpecPublicationGuard, ) -> Result { let bundle = get_os_app(app_name).ok_or_else(|| format!("OS app '{app_name}' not found"))?; let digest = digest_app_bundle(app_name, &bundle); @@ -407,28 +388,22 @@ pub async fn reconcile_os_app( { match ps.get_installed_app(tenant, app_name).await { Ok(Some(record)) => { - let mut specs_ready = tenant_has_ready_app_specs_for_bundle(state, tenant, &bundle); + let specs_ready = tenant_has_ready_app_specs_for_bundle(state, tenant, &bundle); let wasm_registered = tenant_has_registered_wasm_for_bundle(state, tenant, &bundle); let durable_wasm_ready = tenant_has_durable_wasm_for_bundle(state, tenant, &bundle).await; let wasm_ready = wasm_registered && durable_wasm_ready; let policies_active = tenant_has_active_policies_for_bundle(state, tenant, &bundle); - if record.bundle_digest == digest.bundle_digest && !specs_ready { - specs_ready = restore_app_specs_from_matching_digest( - state, - ps.as_ref(), - tenant, - app_name, - &bundle, - ) - .await; - } - - if record.bundle_digest == digest.bundle_digest + if record.status == "installed" + && record.bundle_digest == digest.bundle_digest && specs_ready && wasm_ready && policies_active + && publication_record_override + .as_ref() + .is_none_or(|desired| installed_record_provenance_matches(&record, desired)) + && !state.server.spec_publication_gated(&TenantId::new(tenant)) { tracing::info!( tenant, @@ -442,16 +417,6 @@ pub async fn reconcile_os_app( }); } - if !specs_ready && record.spec_digest == digest.spec_digest { - specs_ready = restore_app_specs_from_matching_digest( - state, - ps.as_ref(), - tenant, - app_name, - &bundle, - ) - .await; - } let plan = plan_reconcile_from_installed_record( &record, &digest, @@ -471,7 +436,15 @@ pub async fn reconcile_os_app( seed = plan.seed, "OS app changed; running delta reconcile" ); - let install = install_os_app_with_plan(state, tenant, app_name, plan).await?; + let install = install_os_app_with_plan_under_guard( + state, + tenant, + app_name, + plan, + publication_record_override, + publication_guard, + ) + .await?; return Ok(OsAppReconcileResult::Installed { app_name: app_name.to_string(), bundle_digest: digest.bundle_digest, @@ -490,8 +463,15 @@ pub async fn reconcile_os_app( } } - let install = - install_os_app_with_plan(state, tenant, app_name, OsAppInstallPlan::all()).await?; + let install = install_os_app_with_plan_under_guard( + state, + tenant, + app_name, + OsAppInstallPlan::all(), + publication_record_override, + publication_guard, + ) + .await?; Ok(OsAppReconcileResult::Installed { app_name: app_name.to_string(), bundle_digest: digest.bundle_digest, @@ -499,15 +479,32 @@ pub async fn reconcile_os_app( }) } +fn installed_record_provenance_matches( + existing: &InstalledAppRecord, + desired: &InstalledAppRecord, +) -> bool { + existing.source_kind == desired.source_kind + && existing.app_ref == desired.app_ref + && existing.version_hash == desired.version_hash + && existing.pinned_version_hash == desired.pinned_version_hash + && existing.current_version_hash == desired.current_version_hash + && existing.follow_policy == desired.follow_policy + && existing.closure_id == desired.closure_id + && existing.registry_url == desired.registry_url + && existing.registry_tenant == desired.registry_tenant +} + /// ARN-68: after a reconcile registers changed specs, re-key any type whose declared /// key-set changed (a newly-declared `[[key]]` on an already-installed type, e.g. /// Directory `ws_path` added after `name_parent`). The once-per-boot startup key-index /// backfill can fire BEFORE this (late — post-boot in prod) reconcile registers the key, /// see only the old key-set, and skip — so the added key would never backfill for /// existing entities and reads scan → 413. Running the key-set-aware re-key here, after -/// registration, closes that race; it only re-scans types whose key-set actually changed -/// (unchanged types skip via the watermark) and is spawned so a large re-key never blocks -/// the reconcile. See ADR-0153. +/// registration, closes that repair gap. Contract activation, exact entity replay, +/// watermark publication, and empty-contract row release happen synchronously around +/// the table swap. This detached pass maintains vector projections and idempotently +/// rechecks key coverage; unchanged ready types reuse their fenced proof without +/// replay. See ADR-0153 and ADR-0192. pub(super) fn spawn_key_index_rekey_after_spec_change(state: &PlatformState, tenant_id: &TenantId) { let server = state.server.clone(); let tenant = tenant_id.clone(); diff --git a/crates/temper-platform/src/os_apps/runtime_heal.rs b/crates/temper-platform/src/os_apps/runtime_heal.rs index aad304590..cf036f953 100644 --- a/crates/temper-platform/src/os_apps/runtime_heal.rs +++ b/crates/temper-platform/src/os_apps/runtime_heal.rs @@ -78,7 +78,7 @@ fn tenant_has_app_spec_tables_for_bundle( .all(|(entity_type, _)| registry.get_table(&tenant_id, entity_type).is_some()) } -fn repair_app_runtime_metadata_from_bundle( +async fn repair_app_runtime_metadata_from_bundle( state: &PlatformState, tenant: &str, app_name: &str, @@ -99,24 +99,54 @@ fn repair_app_runtime_metadata_from_bundle( .map(|(entity_type, ioa_source)| (entity_type.as_str(), ioa_source.as_str())) .collect(); let tenant_id = TenantId::new(tenant); + let target_digest = super::reconcile::digest_app_bundle(app_name, bundle); + let publication_intent = temper_server::ServerState::spec_publication_intent( + "os-app-bundle", + [("bundle", target_digest.bundle_digest.as_bytes())], + ); + let mut publication_guard = state.server.begin_spec_publication(&tenant_id).await?; + if !tenant_has_app_spec_content_for_bundle(state, tenant, bundle) + || !tenant_has_app_spec_tables_for_bundle(state, tenant, bundle) + { + return Err(format!( + "OS app '{app_name}' spec generation changed while runtime repair waited" + )); + } + state + .server + .arm_spec_publication(&mut publication_guard, &tenant_id, &publication_intent)?; + // The matching bundle digest proves these specs are already durable. Fence + // their exact key contracts before restoring the missing live metadata. + let mut cutover = state + .server + .prepare_key_index_contracts_for_spec_activation(&publication_guard, &tenant_id, &specs) + .await?; { let mut registry = state.registry.write().expect("Spec registry lock poisoned"); registry - .try_register_tenant_with_reactions_and_constraints( - tenant_id, + .try_register_tenant_with_reactions_constraints_and_key_epochs( + tenant_id.clone(), csdl, csdl_xml.to_string(), &specs, Vec::new(), bundle.cross_invariants_toml.clone(), true, + &cutover.activation_epochs, ) .map_err(|error| { format!("Failed to restore runtime metadata for os-app '{app_name}': {error}") })?; } + state + .server + .finish_key_index_contract_activation(&mut publication_guard, &tenant_id, &mut cutover) + .await?; state.server.rebuild_reaction_dispatcher(); + state + .server + .complete_spec_publication(&mut publication_guard, &tenant_id)?; tracing::info!( tenant, app = %app_name, @@ -225,7 +255,9 @@ pub(crate) async fn restore_app_specs_from_matching_digest( return false; } - if let Err(error) = repair_app_runtime_metadata_from_bundle(state, tenant, app_name, bundle) { + if let Err(error) = + repair_app_runtime_metadata_from_bundle(state, tenant, app_name, bundle).await + { tracing::warn!( tenant, app = %app_name, diff --git a/crates/temper-platform/src/os_apps/system_files.rs b/crates/temper-platform/src/os_apps/system_files.rs index 327c75c0f..1e86660c5 100644 --- a/crates/temper-platform/src/os_apps/system_files.rs +++ b/crates/temper-platform/src/os_apps/system_files.rs @@ -84,9 +84,10 @@ pub async fn bootstrap_system_files( tenant_id: &TenantId, tenant: &str, system_files: &[SystemFileEntry], -) { + agent_ctx: &temper_server::request_context::AgentContext, +) -> Vec { if system_files.is_empty() { - return; + return Vec::new(); } let has_fs = { @@ -95,20 +96,19 @@ pub async fn bootstrap_system_files( && registry.get_spec(tenant_id, "Directory").is_some() }; if !has_fs { - return; + return Vec::new(); } - let agent_ctx = temper_server::request_context::AgentContext::for_service("platform-bootstrap"); - if let Err(e) = super::ensure_app_docs_workspace(state, tenant_id, &agent_ctx).await { + if let Err(e) = super::ensure_app_docs_workspace(state, tenant_id, agent_ctx).await { tracing::warn!(tenant, error = %e, "Failed to ensure workspace for system file bootstrap"); - return; + return Vec::new(); } // Ensure /system/ root exists. if let Err(e) = ensure_directory( state, tenant_id, - &agent_ctx, + agent_ctx, DirectoryBootstrapTarget { directory_id: "os-system-root", name: "system", @@ -120,11 +120,12 @@ pub async fn bootstrap_system_files( .await { tracing::warn!(tenant, error = %e, "Failed to create /system/ directory for system files"); - return; + return Vec::new(); } // Collect unique subdirectories from relative paths and ensure they exist. let mut ensured_dirs = std::collections::HashSet::new(); + let mut bootstrapped = Vec::new(); for entry in system_files { if let Some(parent) = Path::new(&entry.relative_path).parent() { let parent_str = parent.to_string_lossy().to_string(); @@ -135,7 +136,7 @@ pub async fn bootstrap_system_files( if let Err(e) = ensure_directory( state, tenant_id, - &agent_ctx, + agent_ctx, DirectoryBootstrapTarget { directory_id: &dir_id, name: &parent_str, @@ -171,7 +172,7 @@ pub async fn bootstrap_system_files( match ensure_markdown_file( state, tenant_id, - &agent_ctx, + agent_ctx, MarkdownFileBootstrapTarget { file_id: &file_id, name: &entry.file_name, @@ -185,6 +186,7 @@ pub async fn bootstrap_system_files( { Ok(()) => { tracing::info!(tenant, path = %file_path, "Bootstrapped system file"); + bootstrapped.push(entry.relative_path.clone()); } Err(error) => { tracing::warn!( @@ -196,4 +198,5 @@ pub async fn bootstrap_system_files( } } } + bootstrapped } diff --git a/crates/temper-platform/src/recovery.rs b/crates/temper-platform/src/recovery.rs index 116bbb0dd..a8e69ab70 100644 --- a/crates/temper-platform/src/recovery.rs +++ b/crates/temper-platform/src/recovery.rs @@ -9,9 +9,7 @@ use std::collections::{BTreeMap, BTreeSet}; -use temper_runtime::tenant::TenantId; use temper_server::platform_store::PlatformStore; -use temper_server::registry::VerificationStatus; use crate::os_apps; use crate::state::PlatformState; @@ -52,6 +50,7 @@ pub struct InstalledAppsRuntimeRecoverySummary { pub async fn recover_cedar_policies(state: &PlatformState, ps: &dyn PlatformStore) { let mut legacy_entries: BTreeMap = BTreeMap::new(); let mut granular_entries: BTreeMap> = BTreeMap::new(); + let mut granular_tenants = BTreeSet::new(); match ps.load_tenant_policies().await { Ok(rows) => { @@ -70,6 +69,7 @@ pub async fn recover_cedar_policies(state: &PlatformState, ps: &dyn PlatformStor match ps.load_policy_entries().await { Ok(rows) => { for row in rows { + granular_tenants.insert(row.tenant.clone()); if !row.enabled || row.cedar_text.trim().is_empty() { continue; } @@ -84,13 +84,13 @@ pub async fn recover_cedar_policies(state: &PlatformState, ps: &dyn PlatformStor } } - if legacy_entries.is_empty() && granular_entries.is_empty() { + if legacy_entries.is_empty() && granular_tenants.is_empty() { return; } let tenants: BTreeSet = legacy_entries .keys() - .chain(granular_entries.keys()) + .chain(granular_tenants.iter()) .cloned() .collect(); let mut loaded_count = 0usize; @@ -98,20 +98,19 @@ pub async fn recover_cedar_policies(state: &PlatformState, ps: &dyn PlatformStor let mut skipped_legacy_count = 0usize; for tenant in tenants { let entries = granular_entries.remove(&tenant).unwrap_or_default(); - let has_primary_granular = entries.iter().any(|(policy_id, _)| policy_id == "primary"); let mut policy_text = String::new(); let mut entry_count = 0usize; - // `primary` is the durable aggregate policy row for newer installs. If - // it exists, prefer it over the legacy blob to avoid loading the same - // multi-megabyte generated policy twice. - if !has_primary_granular { - if let Some(legacy_text) = legacy_entries.get(&tenant) { - append_cedar_policy_text(&mut policy_text, legacy_text); - entry_count += 1; + // Row presence establishes the canonical granular generation even when + // every row is disabled. The aggregate is only a compatibility cache and + // must not revive removed or unowned grants during restart. + if granular_tenants.contains(&tenant) { + if legacy_entries.contains_key(&tenant) { + skipped_legacy_count += 1; } - } else if legacy_entries.contains_key(&tenant) { - skipped_legacy_count += 1; + } else if let Some(legacy_text) = legacy_entries.get(&tenant) { + append_cedar_policy_text(&mut policy_text, legacy_text); + entry_count += 1; } for (_, cedar_text) in entries { @@ -119,10 +118,6 @@ pub async fn recover_cedar_policies(state: &PlatformState, ps: &dyn PlatformStor entry_count += 1; } - if policy_text.trim().is_empty() { - continue; - } - // Use the raw policy reload path here instead of per-row PolicyId // rewriting. Production primary policies can contain tens of thousands // of generated statements; raw reload matches the pre-existing startup @@ -198,27 +193,13 @@ pub async fn restore_installed_apps(state: &PlatformState, ps: &dyn PlatformStor | InstalledAppRuntimeRecoveryOutcome::StoreError => {} } - // Legacy recovery still performs a full install when runtime-only - // recovery cannot prove the durable app bundle is unchanged. Startup - // callers that need bounded warm restart should call - // `recover_installed_apps_runtime_state` and then run - // `reconcile_os_app` for their required startup surface. - if tenant_has_ready_app_specs(state, &tenant, &app_name) { - continue; - } - - match os_apps::install_os_app(state, &tenant, &app_name).await { + // Runtime readiness alone cannot prove content, seed, or terminal + // metadata completion. The digest-aware reconciliation plan deliberately + // retries every phase for a non-installed record. + match os_apps::reconcile_os_app(state, &tenant, &app_name).await { Ok(result) => { - let all: Vec = result - .added - .iter() - .chain(&result.updated) - .chain(&result.skipped) - .cloned() - .collect(); tracing::info!( - "Restored app '{app_name}' for '{tenant}': {}", - all.join(", ") + "Reconciled app '{app_name}' for '{tenant}' during recovery: {result:?}" ); } Err(e) => { @@ -250,32 +231,13 @@ pub async fn recover_installed_app_runtime_state( return InstalledAppRuntimeRecoveryOutcome::MissingBundle; }; - let specs_ready = os_apps::tenant_has_ready_app_specs_for_bundle(state, tenant, &bundle); - let policies_active = os_apps::tenant_has_active_policies_for_bundle(state, tenant, &bundle); - let wasm_registered = os_apps::tenant_has_registered_wasm_for_bundle(state, tenant, &bundle); - - if specs_ready && policies_active && wasm_registered { - return InstalledAppRuntimeRecoveryOutcome::Ready; - } - let Some(digest) = os_apps::os_app_bundle_digest(app_name) else { return InstalledAppRuntimeRecoveryOutcome::MissingBundle; }; - match ps.get_installed_app(tenant, app_name).await { - Ok(Some(record)) if record.bundle_digest == digest.bundle_digest => { - let specs_ready = specs_ready - || os_apps::restore_app_specs_from_matching_digest( - state, ps, tenant, app_name, &bundle, - ) - .await; - if specs_ready && policies_active && wasm_registered { - InstalledAppRuntimeRecoveryOutcome::Healed - } else { - InstalledAppRuntimeRecoveryOutcome::NeedsReconcile - } - } - Ok(Some(_)) | Ok(None) => InstalledAppRuntimeRecoveryOutcome::NeedsReconcile, + let record = match ps.get_installed_app(tenant, app_name).await { + Ok(Some(record)) => record, + Ok(None) => return InstalledAppRuntimeRecoveryOutcome::NeedsReconcile, Err(error) => { tracing::warn!( tenant, @@ -283,8 +245,31 @@ pub async fn recover_installed_app_runtime_state( error = %error, "Failed to read installed OS app metadata during runtime recovery" ); - InstalledAppRuntimeRecoveryOutcome::StoreError + return InstalledAppRuntimeRecoveryOutcome::StoreError; } + }; + // Runtime artifacts are not proof that APP.md, skills, seed rows, or final + // metadata committed. Only the terminal install status may use the warm + // readiness fast path. + if record.status != "installed" || record.bundle_digest != digest.bundle_digest { + return InstalledAppRuntimeRecoveryOutcome::NeedsReconcile; + } + + let specs_ready = os_apps::tenant_has_ready_app_specs_for_bundle(state, tenant, &bundle); + let policies_active = os_apps::tenant_has_active_policies_for_bundle(state, tenant, &bundle); + let wasm_registered = os_apps::tenant_has_registered_wasm_for_bundle(state, tenant, &bundle); + + if specs_ready && policies_active && wasm_registered { + return InstalledAppRuntimeRecoveryOutcome::Ready; + } + + let specs_ready = specs_ready + || os_apps::restore_app_specs_from_matching_digest(state, ps, tenant, app_name, &bundle) + .await; + if specs_ready && policies_active && wasm_registered { + InstalledAppRuntimeRecoveryOutcome::Healed + } else { + InstalledAppRuntimeRecoveryOutcome::NeedsReconcile } } @@ -318,151 +303,5 @@ pub async fn recover_installed_apps_runtime_state( summary } -/// Check if all entity types for an app are already registered. -fn tenant_has_ready_app_specs(state: &PlatformState, tenant: &str, app_name: &str) -> bool { - let Some(bundle) = os_apps::get_os_app(app_name) else { - return false; - }; - let tenant_id = TenantId::new(tenant); - let registry = state.registry.read().unwrap(); // ci-ok: infallible lock - bundle.specs.iter().all(|(entity_type, _)| { - let has_table = registry - .get_table(&tenant_id, entity_type.as_str()) - .is_some(); - let is_ready = matches!( - registry.get_verification_status(&tenant_id, entity_type.as_str()), - Some(VerificationStatus::Completed(_) | VerificationStatus::Restored(_)) - ); - has_table && is_ready - }) -} - #[cfg(test)] -mod tests { - use std::collections::HashMap; - - use temper_authz::SecurityContext; - use temper_store_turso::TursoEventStore; - - use super::*; - - fn sqlite_test_url(test_name: &str) -> String { - let mut path = std::env::temp_dir(); - path.push(format!( - "temper-recovery-{test_name}-{}.db", - uuid::Uuid::new_v4() - )); - format!("file:{}", path.display()) - } - - #[tokio::test] - async fn recover_cedar_policies_activates_granular_policy_rows() { - let store = TursoEventStore::new(&sqlite_test_url("granular-policies"), None) - .await - .expect("create test store"); - let policy = r#" -permit( - principal is Agent, - action == Action::"http_call", - resource is HttpEndpoint -) when { - context.module == "build_session_message" -}; -"#; - store - .save_policy("default", "katagami-curation-wasm", policy, "test") - .await - .expect("save granular policy"); - - let state = PlatformState::new(None); - recover_cedar_policies(&state, &store).await; - - let mut resource_attrs = HashMap::new(); - resource_attrs.insert( - "id".to_string(), - serde_json::json!("__trigger__:Submit:build_session_message"), - ); - resource_attrs.insert( - "module".to_string(), - serde_json::json!("build_session_message"), - ); - - let decision = state.server.authz.authorize_for_tenant( - "default", - &SecurityContext::from_resolved_identity("wasm-module", "wasm_module", None), - "http_call", - "HttpEndpoint", - &resource_attrs, - ); - - assert!( - decision.is_allowed(), - "granular policy rows should be active after recovery, got {decision:?}" - ); - assert!( - state - .server - .authz - .get_tenant_policy_text("default") - .expect("tenant policy text") - .contains("build_session_message") - ); - } - - #[tokio::test] - async fn recover_cedar_policies_prefers_primary_row_over_legacy_blob() { - let store = TursoEventStore::new(&sqlite_test_url("primary-policy-recovery"), None) - .await - .expect("create test store"); - store - .upsert_tenant_policy( - "default", - r#"permit(principal, action == Action::"legacy_only", resource);"#, - ) - .await - .expect("save legacy policy"); - store - .save_policy( - "default", - "primary", - r#"permit(principal, action == Action::"read", resource);"#, - "test", - ) - .await - .expect("save primary policy"); - store - .save_policy( - "default", - "katagami-curation-wasm", - r#" -permit( - principal is Agent, - action == Action::"http_call", - resource is HttpEndpoint -) when { - context.module == "build_session_message" -}; -"#, - "test", - ) - .await - .expect("save granular policy"); - - let state = PlatformState::new(None); - recover_cedar_policies(&state, &store).await; - - let tenant_text = state - .server - .authz - .get_tenant_policy_text("default") - .expect("tenant policy text"); - assert!( - tenant_text.contains("build_session_message"), - "granular app policy should be appended to primary policy" - ); - assert!( - !tenant_text.contains("legacy_only"), - "legacy blob should be skipped when durable primary policy row exists" - ); - } -} +mod tests; diff --git a/crates/temper-platform/src/recovery/tests/mod.rs b/crates/temper-platform/src/recovery/tests/mod.rs new file mode 100644 index 000000000..98d2406ab --- /dev/null +++ b/crates/temper-platform/src/recovery/tests/mod.rs @@ -0,0 +1,223 @@ +use std::collections::HashMap; + +use temper_authz::SecurityContext; +use temper_store_turso::TursoEventStore; + +use super::*; + +fn sqlite_test_url(test_name: &str) -> String { + let mut path = std::env::temp_dir(); + path.push(format!( + "temper-recovery-{test_name}-{}.db", + uuid::Uuid::new_v4() + )); + format!("file:{}", path.display()) +} + +#[tokio::test] +async fn recover_cedar_policies_activates_granular_policy_rows() { + let store = TursoEventStore::new(&sqlite_test_url("granular-policies"), None) + .await + .expect("create test store"); + let policy = r#" +permit( + principal is Agent, + action == Action::"http_call", + resource is HttpEndpoint +) when { + context.module == "build_session_message" +}; +"#; + store + .save_policy("default", "katagami-curation-wasm", policy, "test") + .await + .expect("save granular policy"); + + let state = PlatformState::new(None); + recover_cedar_policies(&state, &store).await; + + let mut resource_attrs = HashMap::new(); + resource_attrs.insert( + "id".to_string(), + serde_json::json!("__trigger__:Submit:build_session_message"), + ); + resource_attrs.insert( + "module".to_string(), + serde_json::json!("build_session_message"), + ); + + let decision = state.server.authz.authorize_for_tenant( + "default", + &SecurityContext::from_resolved_identity("wasm-module", "wasm_module", None), + "http_call", + "HttpEndpoint", + &resource_attrs, + ); + + assert!( + decision.is_allowed(), + "granular policy rows should be active after recovery, got {decision:?}" + ); + assert!( + state + .server + .authz + .get_tenant_policy_text("default") + .expect("tenant policy text") + .contains("build_session_message") + ); +} + +#[tokio::test] +async fn recover_cedar_policies_prefers_primary_row_over_legacy_blob() { + let store = TursoEventStore::new(&sqlite_test_url("primary-policy-recovery"), None) + .await + .expect("create test store"); + store + .upsert_tenant_policy( + "default", + r#"permit(principal, action == Action::"legacy_only", resource);"#, + ) + .await + .expect("save legacy policy"); + store + .save_policy( + "default", + "primary", + r#"permit(principal, action == Action::"read", resource);"#, + "test", + ) + .await + .expect("save primary policy"); + store + .save_policy( + "default", + "katagami-curation-wasm", + r#" +permit( + principal is Agent, + action == Action::"http_call", + resource is HttpEndpoint +) when { + context.module == "build_session_message" +}; +"#, + "test", + ) + .await + .expect("save granular policy"); + + let state = PlatformState::new(None); + recover_cedar_policies(&state, &store).await; + + let tenant_text = state + .server + .authz + .get_tenant_policy_text("default") + .expect("tenant policy text"); + assert!( + tenant_text.contains("build_session_message"), + "granular app policy should be appended to primary policy" + ); + assert!( + !tenant_text.contains("legacy_only"), + "legacy blob should be skipped when durable primary policy row exists" + ); +} + +#[tokio::test] +async fn recover_cedar_policies_prefers_any_granular_generation_over_legacy_cache() { + let store = TursoEventStore::new(&sqlite_test_url("owned-policy-recovery"), None) + .await + .expect("create test store"); + store + .upsert_tenant_policy( + "default", + r#"permit(principal, action == Action::"legacy_only", resource);"#, + ) + .await + .expect("save legacy compatibility cache"); + store + .save_policy( + "default", + "owner-a", + r#"permit(principal, action == Action::"granular_only", resource);"#, + "os-app:owner-a", + ) + .await + .expect("save canonical owned row"); + + let state = PlatformState::new(None); + recover_cedar_policies(&state, &store).await; + + let tenant_text = state + .server + .authz + .get_tenant_policy_text("default") + .expect("tenant policy text"); + assert!(tenant_text.contains("granular_only")); + assert!( + !tenant_text.contains("legacy_only"), + "restart must not revive a compatibility aggregate once any granular generation exists" + ); +} + +#[tokio::test] +async fn recover_cedar_policies_clears_previously_active_generation_when_all_rows_disabled() { + let store = TursoEventStore::new(&sqlite_test_url("disabled-policy-recovery"), None) + .await + .expect("create test store"); + let legacy = r#"permit(principal, action == Action::"legacy_only", resource);"#; + store + .upsert_tenant_policy("default", legacy) + .await + .expect("save legacy compatibility cache"); + store + .save_policy( + "default", + "owner-a", + r#"permit(principal, action == Action::"granular_only", resource);"#, + "os-app:owner-a", + ) + .await + .expect("save canonical owned row"); + store + .toggle_policy_enabled("default", "owner-a", false) + .await + .expect("disable canonical row"); + + let state = PlatformState::new(None); + state + .server + .authz + .reload_tenant_policies("default", legacy) + .expect("preload previous generation"); + state + .server + .tenant_policies + .write() + .expect("policy cache lock") + .insert("default".to_string(), legacy.to_string()); + + recover_cedar_policies(&state, &store).await; + + assert_eq!( + state + .server + .authz + .get_tenant_policy_text("default") + .unwrap_or_default(), + "", + "canonical empty generation must revoke the previously active policy" + ); + assert_eq!( + state + .server + .tenant_policies + .read() + .expect("policy cache lock") + .get("default") + .map(String::as_str), + Some("") + ); +} diff --git a/crates/temper-runtime/src/persistence/batch.rs b/crates/temper-runtime/src/persistence/batch.rs new file mode 100644 index 000000000..6c873ce28 --- /dev/null +++ b/crates/temper-runtime/src/persistence/batch.rs @@ -0,0 +1,57 @@ +//! Atomic multi-journal append data types. + +use serde::{Deserialize, Serialize}; + +use super::{EntityKeyRow, PersistenceEnvelope, SnapshotSourceFence}; + +/// Stable, content-bound identity for one atomic multi-journal operation. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PersistenceBatchIdempotency { + /// Durable namespace for the claim, normally the composite parent stream. + pub persistence_id: String, + /// Caller-supplied idempotency key within `persistence_id`. + pub idempotency_key: String, + /// Digest of the complete intended batch; a reused key with different work fails. + pub intent_hash: String, +} + +/// One stream append inside an atomic multi-journal append. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PersistenceAppend { + /// Persistence ID in the form `{tenant}:{entity_type}:{entity_id}`. + pub persistence_id: String, + /// Optimistic-concurrency sequence expected before this append. + pub expected_sequence: u64, + /// Events to append to this journal. + pub events: Vec, + /// The entity's complete current declared-key rows when `reconcile_keys` is true. + #[serde(default)] + pub key_rows: Vec, + /// Whether the batch must replace this entity's complete declared-key row set. + #[serde(default)] + pub reconcile_keys: bool, + /// Versioned declared-key signature used to derive `key_rows`. + #[serde(default)] + pub key_set_signature: Option, + /// Exact snapshot generation used to derive this stream's state and key rows. + #[serde(default)] + pub snapshot_source: SnapshotSourceFence, + /// Optional claim co-committed with the whole append batch. + /// + /// Exactly zero or one append in a batch may carry this value. Stores use + /// it to recognize a committed retry without scanning an unbounded journal. + #[serde(default)] + pub batch_idempotency: Option, +} + +/// New sequence number for one stream after an atomic batch append. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct PersistenceAppendResult { + /// Persistence ID that was appended. + pub persistence_id: String, + /// New highest sequence number for this journal. + pub sequence_nr: u64, + /// Whether the store recognized an already-committed batch claim and made no mutations. + #[serde(default)] + pub batch_already_applied: bool, +} diff --git a/crates/temper-runtime/src/persistence/event_store_index.rs b/crates/temper-runtime/src/persistence/event_store_index.rs new file mode 100644 index 000000000..7c93c2c4b --- /dev/null +++ b/crates/temper-runtime/src/persistence/event_store_index.rs @@ -0,0 +1,426 @@ +//! Declared-key and vector-index methods on the EventStore facade. + +macro_rules! event_store_index_methods { + () => { + /// Whether this backend maintains declared-key rows exactly on every live write, + /// can repair them from replay, and persists coverage watermarks. Only such a + /// backend may answer keyed hits/misses as an authoritative ownership oracle. + /// Defaults to false so no-op index methods can never create false authority. + fn supports_authoritative_key_index(&self) -> bool { + false + } + + /// Append events to the journal. + fn append( + &self, + persistence_id: &str, + expected_sequence: u64, + events: &[PersistenceEnvelope], + ) -> impl std::future::Future> + Send; + + /// Append events and co-commit declared key-index rows (ADR-0153) in the + /// **same transaction** as the journal append. `key_rows` is the entity's exact + /// current key set, including an empty set after delete or key removal. A thin + /// forwarder to [`EventStore::append_with_index_rows`] with key reconciliation + /// enabled and no vector rows. The co-commit logic lives in + /// `append_with_index_rows`, which query-plane backends override. + fn append_with_keys( + &self, + persistence_id: &str, + expected_sequence: u64, + events: &[PersistenceEnvelope], + key_rows: &[EntityKeyRow], + ) -> impl std::future::Future> + Send { + self.append_with_index_rows( + persistence_id, + expected_sequence, + events, + key_rows, + &[], + IndexReconciliation { + keys: true, + key_set_signature: None, + vectors: false, + snapshot_source: SnapshotSourceFence::Unchecked, + }, + ) + } + + /// Append events and co-commit BOTH declared key-index rows (ADR-0153) and + /// derived vector-index rows (ADR-0155) in the **same transaction** as the + /// journal append. This is the single co-commit entry point the entity actor + /// calls. The default ignores the index kinds and delegates to + /// [`EventStore::append`] only when the snapshot source is unchecked. A + /// snapshot-fenced append fails closed unless the backend overrides this + /// method and validates the source atomically with the journal write. Stores + /// with a query plane that co-commit (postgres, sim) override it; Turso also + /// overrides it to maintain the vector index write-behind (event first, index + /// follows). When `reconciliation.keys` is true + /// (the entity's type declares ≥1 `[[key]]`) the store validates every current + /// claim, then DELETES all prior key rows for the entity and inserts `key_rows` in + /// the journal transaction. Empty rows therefore release ownership. Likewise, + /// when `reconciliation.vectors` is true (the entity's type declares ≥1 + /// `[[vector]]` path) the store first DELETES all of the entity's vector rows, + /// then inserts `vector_rows`. The sequence and atomicity contract is identical + /// to `append`. + fn append_with_index_rows( + &self, + persistence_id: &str, + expected_sequence: u64, + events: &[PersistenceEnvelope], + key_rows: &[EntityKeyRow], + vector_rows: &[EntityVectorRow], + reconciliation: IndexReconciliation, + ) -> impl std::future::Future> + Send { + let _ = (key_rows, vector_rows); + async move { + if !matches!( + reconciliation.snapshot_source, + SnapshotSourceFence::Unchecked + ) { + return Err(PersistenceError::SnapshotGenerationChanged); + } + self.append(persistence_id, expected_sequence, events).await + } + } + + /// Reconcile the derived vector-index rows for an **existing** entity to exactly + /// `vector_rows` (ADR-0155), without appending a journal event: DELETE every + /// existing row for `(tenant, entity_type, entity_id)`, then INSERT `vector_rows`. + /// Idempotent, and an empty `vector_rows` PURGES the entity (used to clean up a + /// deleted or un-embedded entity). Used by the backfill and by the Turso + /// write-behind path. The default is a no-op (non-indexing backends); query-plane + /// stores implement it. + fn backfill_entity_vectors( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + vector_rows: &[EntityVectorRow], + ) -> impl std::future::Future> + Send { + let _ = (tenant, entity_type, entity_id, vector_rows); + async { Ok(()) } + } + + /// The candidate `(entity_id, vector)` rows for one vector-index partition + /// `(tenant, entity_type, decl_name, model_tag)`, in **deterministic entity-id + /// order** (ADR-0155), capped at `limit` rows. The kernel ranks these; the store + /// only supplies the packed vectors, and applies `LIMIT` so an over-budget + /// partition is detected (caller passes `budget + 1`) without loading the whole + /// partition into memory. Default empty (non-indexing backends have no index). + fn vector_candidates( + &self, + tenant: &str, + entity_type: &str, + decl_name: &str, + model_tag: &str, + limit: usize, + ) -> impl std::future::Future, PersistenceError>> + Send + { + let _ = (tenant, entity_type, decl_name, model_tag, limit); + async { Ok(Vec::new()) } + } + + /// Record that `entity_vector_index` is **complete** for `(tenant, entity_type)` + /// — every existing entity has had its declared vectors indexed by the backfill + /// (ADR-0155 watermark, mirroring `mark_key_index_backfilled`). `vector_set` is + /// the sorted, comma-joined declared vector-path NAMES the backfill covered, so a + /// later declaration of an ADDITIONAL path is detected as a set change and the + /// type is re-indexed. Idempotent. Default no-op. + fn mark_vector_index_backfilled( + &self, + tenant: &str, + entity_type: &str, + vector_set: &str, + ) -> impl std::future::Future> + Send { + let _ = (tenant, entity_type, vector_set); + async { Ok(()) } + } + + /// The `(entity_type, vector_set)` watermarks for `tenant` — each type whose + /// `entity_vector_index` backfill is complete, paired with the covered path set. + /// Default empty (no backend authority). Mirrors `key_index_backfilled_types`. + fn vector_index_backfilled_types( + &self, + tenant: &str, + ) -> impl std::future::Future, PersistenceError>> + Send + { + let _ = tenant; + async { Ok(Vec::new()) } + } + + /// The `entity_id`s that already have at least one `entity_vector_index` row for + /// `(tenant, entity_type)`. Lets the vector backfill **resume** cheaply, skipping + /// already-indexed entities. Default empty (no resumption). Mirrors + /// `keyed_entity_ids_for_type`. + fn vectored_entity_ids_for_type( + &self, + tenant: &str, + entity_type: &str, + ) -> impl std::future::Future, PersistenceError>> + Send { + let _ = (tenant, entity_type); + async { Ok(Vec::new()) } + } + + /// Reconcile declared key-index rows for an **existing** entity (ADR-0153, + /// ADR-0192), without appending a journal event. The store first validates that + /// `contract_fence` still identifies the tenant/type contract, exact journal + /// boundary, exact snapshot generation, and entity liveness under which replay + /// derived `key_rows`, then + /// validates `expected_sequence`. Only while all fences hold does it DELETE every existing + /// row for `(tenant, entity_type, entity_id)` and INSERT `key_rows`. Idempotent, + /// and an empty set purges stale ownership for deleted or currently unkeyable + /// entities. A concurrent type-contract change fails with + /// [`PersistenceError::KeyContractChanged`]; a concurrent journal-source change + /// fails with [`PersistenceError::JournalBoundaryChanged`]; a concurrent liveness + /// change fails with [`PersistenceError::EntityLivenessChanged`]; a concurrent + /// snapshot change fails with [`PersistenceError::SnapshotGenerationChanged`]; a concurrent + /// durable sequence advance fails with [`PersistenceError::ConcurrencyViolation`]. Used to populate and repair + /// `entity_key_index` before a keyed read can treat absence as authoritative. The + /// default is a no-op (non-indexing backends). + fn backfill_entity_keys( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + expected_sequence: u64, + contract_fence: KeyIndexBackfillFence<'_>, + key_rows: &[EntityKeyRow], + ) -> impl std::future::Future> + Send { + let _ = ( + tenant, + entity_type, + entity_id, + expected_sequence, + contract_fence, + key_rows, + ); + async { Ok(()) } + } + + /// Resolve an entity by a declared key (ADR-0153): the `entity_id` currently + /// holding `(key_name, key_hash)`, or `None` if absent. This is the + /// negative-existence access path — present *and* absent in one `O(log n)` + /// probe, no scan. Default returns `None` (non-indexing backends); the + /// query-plane stores override it against `entity_key_index`. + fn lookup_by_key( + &self, + tenant: &str, + entity_type: &str, + key_name: &str, + key_hash: &str, + ) -> impl std::future::Future, PersistenceError>> + Send { + let _ = (tenant, entity_type, key_name, key_hash); + async { Ok(None) } + } + + /// Resolve a declared-key owner together with its co-committed journal + /// generation. Authoritative backends override this from the same row used + /// by [`EventStore::lookup_by_key`]. The compatibility default preserves + /// non-authoritative/custom stores while assigning generation zero. + fn lookup_by_key_with_sequence( + &self, + tenant: &str, + entity_type: &str, + key_name: &str, + key_hash: &str, + ) -> impl std::future::Future, PersistenceError>> + Send + { + async move { + self.lookup_by_key(tenant, entity_type, key_name, key_hash) + .await + .map(|owner| { + owner.map(|entity_id| EntityKeyLookup { + entity_id, + sequence_nr: 0, + }) + }) + } + } + + /// Record that `entity_key_index` is **complete** for `(tenant, entity_type)` + /// — every existing entity of that type has been keyed by the backfill + /// (ADR-0153/0171 watermark). Only once set may a keyed read trust an indexed + /// hit or miss. A miss is then authoritative absence, retiring the full-type + /// reconcile scan (#324) for that type. Idempotent. + /// + /// **Soundness invariant — only override this on a backend that co-commits key + /// rows on EVERY write** (i.e. overrides [`EventStore::append_with_keys`]). The + /// watermark asserts the index is complete *and stays complete*; a backend that + /// backfills but does not maintain keys live (e.g. Turso, which does not + /// co-commit) would let a later write go unkeyed, and a keyed miss for that + /// present entity would then read as authoritative absence — a silent + /// correctness bug. Such backends MUST keep the default no-op so they never + /// become authoritative (their keyed misses fall back to the scan — correct, + /// just not bounded). Postgres co-commits and overrides this; the sim store does + /// too for DST. The default is a no-op. + /// + /// `key_set` is the derivation-contract version plus every sorted declaration's + /// name and ordered properties. It is recorded so either a definition change or + /// a reconciliation-semantics upgrade forces a complete repair pass. + fn mark_key_index_backfilled( + &self, + tenant: &str, + entity_type: &str, + key_set: &str, + ) -> impl std::future::Future> + Send { + let _ = (tenant, entity_type, key_set); + async { Ok(()) } + } + + /// The `(entity_type, key_set)` watermarks for `tenant` — each type whose + /// `entity_key_index` backfill is complete, paired with the versioned declared-key + /// signature it covered. The read plane caches these so a keyed hit or miss is + /// authoritative ONLY when the covered signature still equals the current one. + /// Default empty (no backend authority → scan-safe). + fn key_index_backfilled_types( + &self, + tenant: &str, + ) -> impl std::future::Future, PersistenceError>> + Send + { + let _ = tenant; + async { Ok(Vec::new()) } + } + + /// Every durable `(tenant, entity_type)` with an activated key contract. + /// + /// Startup uses this authority to retire contracts whose replace-mode spec + /// deletion committed before the later runtime activation transaction. + fn key_index_activated_contracts( + &self, + ) -> impl std::future::Future, PersistenceError>> + Send + { + let authoritative = self.supports_authoritative_key_index(); + async move { + if authoritative { + return Err(PersistenceError::Storage( + "authoritative key store did not implement activated-contract enumeration" + .to_string(), + )); + } + Ok(Vec::new()) + } + } + + /// Monotonic revision of the live declared-key reconciliation universe for one + /// type. It changes when a durable write uses a different key signature or when + /// a snapshot/catalog-only mutation is not already represented by the journal. + /// Backfill captures this before enumeration and conditionally publishes its + /// watermark against it, so neither a contract race nor a newly durable entity + /// can certify stale coverage. + fn key_index_reconciliation_revision( + &self, + tenant: &str, + entity_type: &str, + ) -> impl std::future::Future> + Send { + let _ = (tenant, entity_type); + async { Ok(0) } + } + + /// Establish `key_set` as the target contract before a full backfill starts and + /// return its monotonic revision. This invalidates older coverage up front. A + /// concurrent live write under a different signature must then advance the + /// revision, causing the final conditional watermark publication to fail. + fn begin_key_index_backfill( + &self, + tenant: &str, + entity_type: &str, + key_set: &str, + ) -> impl std::future::Future> + Send { + let _ = (tenant, entity_type, key_set); + async { Ok(0) } + } + + /// Establish a key contract before its spec becomes live. When + /// `purge_existing_rows` is true, the contract change and type-wide removal + /// of prior ownership rows must be atomic under the same contract fence. + fn activate_key_index_contract( + &self, + tenant: &str, + entity_type: &str, + key_set: &str, + purge_existing_rows: bool, + ) -> impl std::future::Future> + Send { + async move { + if self.supports_authoritative_key_index() { + return Err(PersistenceError::Storage(format!( + "authoritative key store did not implement fenced contract activation for {tenant}:{entity_type}" + ))); + } + let revision = self + .begin_key_index_backfill(tenant, entity_type, key_set) + .await?; + let _ = purge_existing_rows; + Ok(revision) + } + } + + /// Atomically activate every changed entity type in one spec publication + /// and return the monotonic epoch assigned to each type. Authoritative + /// backends must override multi-type activation so a failure cannot leave a + /// partially fenced tenant while the old registry remains live. + fn activate_key_index_contracts( + &self, + tenant: &str, + activations: &[KeyContractActivation], + ) -> impl std::future::Future< + Output = Result, PersistenceError>, + > + Send { + async move { + if self.supports_authoritative_key_index() { + return Err(PersistenceError::Storage(format!( + "authoritative key store did not implement atomic spec-fingerprinted activation for {tenant}" + ))); + } + let mut epochs = std::collections::BTreeMap::new(); + for activation in activations { + let epoch = self + .activate_key_index_contract( + tenant, + &activation.entity_type, + &activation.key_set, + activation.purge_existing_rows, + ) + .await?; + epochs.insert(activation.entity_type.clone(), epoch); + } + Ok(epochs) + } + } + + /// Publish a coverage watermark only if the type's live key-contract revision is + /// still `expected_revision` AND its signature is still `key_set`. Returns false + /// when a concurrent write changed either; callers must leave the type scan-safe + /// and retry a full repair. + fn mark_key_index_backfilled_if_revision( + &self, + tenant: &str, + entity_type: &str, + key_set: &str, + expected_revision: u64, + ) -> impl std::future::Future> + Send { + let _ = expected_revision; + async move { + self.mark_key_index_backfilled(tenant, entity_type, key_set) + .await?; + Ok(true) + } + } + + /// The `entity_id`s that already have at least one `entity_key_index` row for + /// `(tenant, entity_type)`. Retained for index inspection and backend conformance; + /// exact ADR-0192 repair does not use presence as proof of completeness because + /// an existing row may itself be stale. Default empty. + fn keyed_entity_ids_for_type( + &self, + tenant: &str, + entity_type: &str, + ) -> impl std::future::Future, PersistenceError>> + Send { + let _ = (tenant, entity_type); + async { Ok(Vec::new()) } + } + + }; +} + +pub(crate) use event_store_index_methods; diff --git a/crates/temper-runtime/src/persistence/event_store_journal.rs b/crates/temper-runtime/src/persistence/event_store_journal.rs new file mode 100644 index 000000000..9a40ad11b --- /dev/null +++ b/crates/temper-runtime/src/persistence/event_store_journal.rs @@ -0,0 +1,255 @@ +//! Journal, snapshot, and entity-enumeration methods on the EventStore facade. + +macro_rules! event_store_journal_methods { + () => { + /// Atomically append events to multiple journals. + /// + /// Backends must either commit every append in `appends`, or commit none. For + /// each item whose `reconcile_keys` is true, its `key_rows` are the exact final + /// declared-key set and must change atomically with every journal in the batch. + /// This is the storage primitive composite actions use for one physical unit. + fn append_batch( + &self, + appends: &[PersistenceAppend], + ) -> impl std::future::Future, PersistenceError>> + Send; + + /// Check whether a content-bound atomic batch claim already committed. + /// + /// Returns `true` only for an exact intent match, `false` when the claim is + /// absent, and an error when the same namespace/key names different work. + /// Composite retry paths call this before current-state validation so an + /// aged stream cannot hide a durable replay behind bounded actor history. + fn batch_idempotency_committed( + &self, + claim: &PersistenceBatchIdempotency, + ) -> impl std::future::Future> + Send { + let _ = claim; + async { Ok(false) } + } + + /// Read events from the journal, starting after the given sequence number. + fn read_events( + &self, + persistence_id: &str, + from_sequence: u64, + ) -> impl std::future::Future, PersistenceError>> + Send; + + /// Read one bounded, ascending journal page after `from_sequence` and no later + /// than the inclusive `through_sequence` captured by the caller. + /// + /// Implementations must apply `limit` at the storage boundary rather than + /// fetching an unbounded suffix first. Callers must pass a positive limit. + fn read_events_page( + &self, + persistence_id: &str, + from_sequence: u64, + through_sequence: u64, + limit: usize, + ) -> impl std::future::Future, PersistenceError>> + Send; + + /// Return the exact durable high-water and terminal lifecycle boundary for one + /// stream. + /// + /// Snapshots are derived acceleration state and cannot outrank a terminal + /// journal event. Recovery also checks the high-water after replay so a + /// fault-truncated prefix cannot be accepted as current state. Backends whose + /// normal reads may be truncated must override this compatibility scan with an + /// exact metadata lookup. + fn journal_boundary( + &self, + persistence_id: &str, + ) -> impl std::future::Future> + Send { + async move { + let events = self.read_events(persistence_id, 0).await?; + Ok(JournalBoundary { + latest_sequence: events.last().map(|event| event.sequence_nr).unwrap_or(0), + first_terminal_sequence: events + .iter() + .find(|event| event.transitions_to_deleted()) + .map(|event| event.sequence_nr), + }) + } + } + + /// Return the first durable terminal sequence, if any. + /// + /// This compatibility helper delegates to [`EventStore::journal_boundary`]; + /// recovery should use the complete boundary so it can also prove replay + /// completeness. + fn terminal_tombstone_sequence( + &self, + persistence_id: &str, + ) -> impl std::future::Future, PersistenceError>> + Send { + async move { + Ok(self + .journal_boundary(persistence_id) + .await? + .first_terminal_sequence) + } + } + + /// Save a state snapshot without regressing the current generation. + /// + /// An older sequence and an identical same-sequence write are no-ops. A + /// same-sequence write with different bytes replaces that source generation; + /// a newer sequence advances it. The accept/no-op decision must cover every + /// snapshot-derived mutation atomically. + fn save_snapshot( + &self, + persistence_id: &str, + sequence_nr: u64, + snapshot: &[u8], + ) -> impl std::future::Future> + Send; + + /// Save a state snapshot only while its exact derivation source is current. + /// + /// Implementations must validate `source` and apply the snapshot write in one + /// stream-fenced transaction. The conservative default supports only + /// [`SnapshotSourceFence::Unchecked`]; stores used by entity actors override + /// this method so stale passivation and queued writers cannot replace a + /// same-sequence snapshot generation. + fn save_snapshot_if_source( + &self, + persistence_id: &str, + sequence_nr: u64, + snapshot: &[u8], + source: &SnapshotSourceFence, + key_contract: Option<&str>, + ) -> impl std::future::Future> + Send { + async move { + let _ = key_contract; + if !matches!(source, SnapshotSourceFence::Unchecked) { + return Err(PersistenceError::SnapshotGenerationChanged); + } + self.save_snapshot(persistence_id, sequence_nr, snapshot) + .await + } + } + + /// Load the latest snapshot. + fn load_snapshot( + &self, + persistence_id: &str, + ) -> impl std::future::Future)>, PersistenceError>> + Send; + + /// List all distinct `(entity_type, entity_id)` pairs for a tenant. + fn list_entity_ids( + &self, + tenant: &str, + ) -> impl std::future::Future, PersistenceError>> + Send; + + /// List distinct entity IDs for one `(tenant, entity_type)` pair. + fn list_entity_ids_by_type( + &self, + tenant: &str, + entity_type: &str, + ) -> impl std::future::Future, PersistenceError>> + Send; + + /// List every stream or derived-key owner that must participate in exact key + /// reconciliation for one `(tenant, entity_type)`. Unlike normal live-entity + /// enumeration, this includes deleted journal streams and key-index-only + /// phantoms so repair can purge their stale ownership before watermarking. + /// Backends without a separate authoritative key index inherit the ordinary + /// type enumeration. + fn list_entity_ids_for_key_reconciliation( + &self, + tenant: &str, + entity_type: &str, + ) -> impl std::future::Future, PersistenceError>> + Send { + self.list_entity_ids_by_type(tenant, entity_type) + } + + /// Capture the inclusive terminal entity ID for one bounded repair scan. + /// Candidates created after this boundary are exact current-contract writes + /// and must not extend an already-running traversal indefinitely. + fn key_reconciliation_boundary( + &self, + tenant: &str, + entity_type: &str, + ) -> impl std::future::Future, PersistenceError>> + Send { + async move { + Ok(self + .list_entity_ids_for_key_reconciliation(tenant, entity_type) + .await? + .into_iter() + .max()) + } + } + + /// Read one deterministic, storage-bounded page of declared-key repair + /// candidates after `after_entity_id`. + /// + /// Authoritative key stores must override this and derive `is_live` in the + /// same storage query as the candidate page. The final reconciliation- + /// revision CAS invalidates every page if a source changes during traversal. + fn list_key_reconciliation_page( + &self, + tenant: &str, + entity_type: &str, + after_entity_id: Option<&str>, + through_entity_id: &str, + limit: usize, + ) -> impl std::future::Future, PersistenceError>> + Send + { + async move { + assert!(limit > 0, "key reconciliation page limit must be positive"); + if self.supports_authoritative_key_index() { + return Err(PersistenceError::Storage(format!( + "authoritative key store did not implement bounded repair paging for {tenant}:{entity_type}" + ))); + } + let mut entity_ids = self + .list_entity_ids_for_key_reconciliation(tenant, entity_type) + .await?; + entity_ids.sort(); + entity_ids.dedup(); + Ok(entity_ids + .into_iter() + .filter(|entity_id| { + after_entity_id.is_none_or(|cursor| entity_id.as_str() > cursor) + && entity_id.as_str() <= through_entity_id + }) + .take(limit) + .map(|entity_id| KeyReconciliationEntity { + entity_id, + is_live: true, + }) + .collect()) + } + } + + /// List at most `limit` authoritative `(entity_type, entity_id)` pairs for + /// a tenant, optionally scoped to one entity type. + /// + /// Storage backends should override this to apply the bound inside the + /// backing query. The default is intended for small in-memory/test stores. + fn list_entity_ids_limited( + &self, + tenant: &str, + entity_type: Option<&str>, + limit: usize, + ) -> impl std::future::Future, PersistenceError>> + Send + { + async move { + if limit == 0 { + return Ok(Vec::new()); + } + let mut entities = if let Some(entity_type) = entity_type { + self.list_entity_ids_by_type(tenant, entity_type) + .await? + .into_iter() + .map(|entity_id| (entity_type.to_string(), entity_id)) + .collect::>() + } else { + self.list_entity_ids(tenant).await? + }; + entities.sort(); + entities.truncate(limit); + Ok(entities) + } + } + + }; +} + +pub(crate) use event_store_journal_methods; diff --git a/crates/temper-runtime/src/persistence/index.rs b/crates/temper-runtime/src/persistence/index.rs new file mode 100644 index 000000000..cea49a1e1 --- /dev/null +++ b/crates/temper-runtime/src/persistence/index.rs @@ -0,0 +1,203 @@ +//! Derived key/vector index contracts shared by event-store backends. + +use serde::{Deserialize, Serialize}; + +const ACTIVATED_KEY_CONTRACT_PREFIX: &str = "@temper-key-contract-v1:"; + +/// Attach the monotonic spec-activation epoch captured by a durable writer to +/// its stable declared-key signature. +pub fn encode_activated_key_contract(key_set: &str, activation_epoch: u64) -> String { + format!( + "{ACTIVATED_KEY_CONTRACT_PREFIX}{activation_epoch}:{}:{key_set}", + key_set.len() + ) +} + +/// Split a writer contract into its stable key-set signature and optional +/// activation epoch. Plain legacy/backfill signatures have no epoch. +pub fn decode_activated_key_contract(contract: &str) -> (&str, Option) { + let Some(encoded) = contract.strip_prefix(ACTIVATED_KEY_CONTRACT_PREFIX) else { + return (contract, None); + }; + let Some((epoch, remainder)) = encoded.split_once(':') else { + return (contract, None); + }; + let Some((length, key_set)) = remainder.split_once(':') else { + return (contract, None); + }; + let Ok(epoch) = epoch.parse::() else { + return (contract, None); + }; + let Ok(length) = length.parse::() else { + return (contract, None); + }; + if key_set.len() != length { + return (contract, None); + } + (key_set, Some(epoch)) +} + +/// One entity type's declared-key contract in an atomic spec activation batch. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct KeyContractActivation { + /// Entity type whose live table will receive the returned epoch. + pub entity_type: String, + /// Stable declared-key-set signature (without an activation epoch). + pub key_set: String, + /// Stable fingerprint of the full IOA source whose replay semantics derive + /// key-property fields. A change forces coverage invalidation even when the + /// declared key set itself is byte-identical. + pub spec_fingerprint: String, + /// Whether activation must atomically purge every prior ownership row for + /// the type (the exact empty-contract transition). + pub purge_existing_rows: bool, +} + +/// A declared-key row to co-commit with an append (ADR-0153). The entity claims +/// `key_hash` for `key_name`; the store writes it into `entity_key_index` in the +/// same transaction as the journal append. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct EntityKeyRow { + /// The declared key's identifier (the `[[key]]` block's `name`). + pub key_name: String, + /// The canonical, type-tagged hash of the key's values. + pub key_hash: String, +} + +/// Authoritative owner and journal generation for one declared-key row. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EntityKeyLookup { + /// Entity that currently owns the declared key. + pub entity_id: String, + /// Owner journal sequence co-committed with this key row. + pub sequence_nr: u64, +} + +/// Contract and entity classification captured before a declared-key backfill +/// replays entity state. Every repair row must validate this fence before mutation +/// (ADR-0192). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct KeyIndexBackfillFence<'a> { + /// Versioned signature used to derive the attempted repair rows. + pub key_set_signature: &'a str, + /// Monotonic contract revision captured when the repair pass began. + pub contract_revision: u64, + /// Exact journal high-water observed before reconstructing the repair row. + /// Revalidating this separately from the newest derived sequence prevents a + /// catalog/snapshot-only owner at generation N from racing a first journal + /// append at the same numeric generation N. + pub expected_journal_sequence: u64, + /// Whether authoritative enumeration classified the entity as live. The store + /// revalidates this under the same stream lock as exact key reconciliation. + pub expected_entity_live: bool, + /// Exact snapshot generation whose legacy fields participated in state + /// reconstruction. Presence, sequence, and bytes are revalidated together; + /// `None` proves that no snapshot participated. + pub expected_snapshot: Option>, +} + +/// Exact derived snapshot generation used as a legacy baseline during key repair. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct SnapshotBackfillFence<'a> { + /// Snapshot sequence captured before reconstruction. + pub sequence_nr: u64, + /// Exact serialized snapshot bytes captured before reconstruction. + pub state: &'a [u8], +} + +/// Exact journal and snapshot generation used to derive a query projection. +/// +/// Projection repair must validate both components in the same storage +/// transaction that mutates the catalog and field index. Comparing only a +/// numeric sequence cannot distinguish a same-sequence snapshot replacement. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct ProjectionSourceFence<'a> { + /// Exact journal high-water captured during state reconstruction. + pub expected_journal_sequence: u64, + /// Exact snapshot generation that participated in reconstruction. `None` + /// proves that no snapshot was present. + pub expected_snapshot: Option>, +} + +/// Exact durable snapshot generation an event append was derived from. +/// +/// `Unchecked` preserves compatibility for callers whose state does not depend on +/// snapshots. Entity actors and atomic composite writers must instead use +/// `Absent` or `Exact`, allowing the store to reject a same-sequence snapshot +/// replacement before journal and derived-index rows are mutated. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub enum SnapshotSourceFence { + /// The caller does not participate in snapshot-source fencing. + #[default] + Unchecked, + /// State reconstruction observed no durable snapshot. + Absent, + /// State reconstruction used this exact snapshot sequence and payload. + Exact { + /// Snapshot sequence captured during state reconstruction. + sequence_nr: u64, + /// Exact serialized snapshot payload captured during reconstruction. + state: Vec, + }, +} + +/// A derived vector-index row to co-commit with an append (ADR-0155). +#[derive(Debug, Clone, PartialEq)] +pub struct EntityVectorRow { + /// The declared vector path's identifier (the `[[vector]]` block's `name`). + pub decl_name: String, + /// The model tag that partitions this vector's space. + pub model_tag: String, + /// The float vector, exactly `dims` long. + pub vector: Vec, +} + +/// Which derived index families an append authoritatively reconciles to the supplied +/// exact row sets. A participating store ignores a row family when its flag is false; +/// when true, even an empty row set means "delete every prior row for this entity." +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct IndexReconciliation { + /// Reconcile the entity's complete declared-key ownership set (ADR-0192). + pub keys: bool, + /// Versioned signature of the complete declared-key contract used for this write. + pub key_set_signature: Option, + /// Reconcile the entity's complete derived-vector row set (ADR-0155). + pub vectors: bool, + /// Exact snapshot generation used to derive the event and index rows. + pub snapshot_source: SnapshotSourceFence, +} + +/// Pack an `f32` slice to little-endian bytes for `entity_vector_index`. +pub fn pack_f32_le(vector: &[f32]) -> Vec { + let mut bytes = Vec::with_capacity(vector.len() * 4); + for value in vector { + bytes.extend_from_slice(&value.to_le_bytes()); + } + bytes +} + +/// Unpack finite little-endian `f32` values, rejecting corrupt byte lengths and +/// non-finite components. +pub fn unpack_f32_le(bytes: &[u8]) -> Option> { + if !bytes.len().is_multiple_of(4) { + return None; + } + let mut out = Vec::with_capacity(bytes.len() / 4); + for chunk in bytes.chunks_exact(4) { + let value = f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]); + if !value.is_finite() { + return None; + } + out.push(value); + } + Some(out) +} + +/// One entity/vector candidate returned from an index partition. +#[derive(Debug, Clone, PartialEq)] +pub struct EntityVectorCandidate { + /// The entity holding this vector. + pub entity_id: String, + /// The float vector, exactly `dims` long. + pub vector: Vec, +} diff --git a/crates/temper-runtime/src/persistence/materialization.rs b/crates/temper-runtime/src/persistence/materialization.rs new file mode 100644 index 000000000..9694ae586 --- /dev/null +++ b/crates/temper-runtime/src/persistence/materialization.rs @@ -0,0 +1,165 @@ +//! Bounded state-materialization control records. + +use serde::Serialize; + +use super::PersistenceEnvelope; + +/// Reserved first-journal event that transfers a snapshot-only entity into a +/// self-contained journal generation. +/// +/// Stores use this control record, together with an exact snapshot-source fence, +/// to retire the migration snapshot in the same atomic append and to reject a +/// delayed ahead-of-journal snapshot from recreating that retired generation. +pub const STATE_MATERIALIZATION_EVENT_TYPE: &str = "Temper.Internal.StateMaterialization.v1"; + +/// Payload schema that distinguishes the runtime control record from a legal +/// domain action with the same event-type string. +pub const STATE_MATERIALIZATION_SCHEMA: &str = "temper.state-materialization.v1"; + +/// Maximum durable idempotency entries accepted in a state-materialization +/// control record. +pub const STATE_MATERIALIZATION_IDEMPOTENCY_KEY_BUDGET: usize = 1_000; + +/// Maximum serialized bytes accepted for a state-materialization control payload. +/// +/// The control record is created only when a legacy snapshot-only generation +/// first enters the journal. Bounding the one-time baseline prevents an +/// arbitrarily large snapshot from being cloned into an unbounded event. +pub const STATE_MATERIALIZATION_PAYLOAD_BYTE_BUDGET: usize = 16 * 1024 * 1024; + +struct JsonByteBudget { + remaining: usize, + exhausted: bool, +} + +impl std::io::Write for JsonByteBudget { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + if bytes.len() > self.remaining { + self.exhausted = true; + return Err(std::io::Error::other("JSON byte budget exhausted")); + } + self.remaining -= bytes.len(); + Ok(bytes.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +/// Check a value's serialized JSON size without allocating its serialized form. +pub fn serialized_json_fits_byte_budget( + value: &T, + byte_budget: usize, +) -> Result { + let mut writer = JsonByteBudget { + remaining: byte_budget, + exhausted: false, + }; + match serde_json::to_writer(&mut writer, value) { + Ok(()) => Ok(true), + Err(_) if writer.exhausted => Ok(false), + Err(error) => Err(error.to_string()), + } +} + +fn object_values_match( + value: Option<&serde_json::Value>, + predicate: impl Fn(&serde_json::Value) -> bool, +) -> bool { + value + .and_then(serde_json::Value::as_object) + .is_some_and(|entries| entries.values().all(predicate)) +} + +/// Return whether an event type and payload form the internal snapshot-to- +/// journal materialization control record for the specified stream identity. +/// +/// Event type alone is deliberately insufficient: specs may legally declare an +/// action with the same name. Stores use this exact discriminator before +/// retiring a migration snapshot or suppressing a delayed snapshot writer. +pub fn is_state_materialization_payload_for( + event_type: &str, + payload: &serde_json::Value, + entity_type: &str, + entity_id: &str, +) -> bool { + if event_type != STATE_MATERIALIZATION_EVENT_TYPE + || payload.get("schema").and_then(serde_json::Value::as_str) + != Some(STATE_MATERIALIZATION_SCHEMA) + || !matches!( + serialized_json_fits_byte_budget(payload, STATE_MATERIALIZATION_PAYLOAD_BYTE_BUDGET), + Ok(true) + ) + { + return false; + } + let Some(state) = payload.get("state").and_then(serde_json::Value::as_object) else { + return false; + }; + let Some(status) = state.get("status").and_then(serde_json::Value::as_str) else { + return false; + }; + let fields_match = state + .get("fields") + .and_then(serde_json::Value::as_object) + .is_some_and(|fields| { + fields.get("Id").and_then(serde_json::Value::as_str) == Some(entity_id) + && fields.get("Status").and_then(serde_json::Value::as_str) == Some(status) + }); + let processed_keys_match = state + .get("processed_idempotency_keys") + .and_then(serde_json::Value::as_object) + .is_some_and(|entries| { + entries.len() <= STATE_MATERIALIZATION_IDEMPOTENCY_KEY_BUDGET + && entries.values().all(|value| value.as_u64().is_some()) + }); + + state.get("entity_type").and_then(serde_json::Value::as_str) == Some(entity_type) + && state.get("entity_id").and_then(serde_json::Value::as_str) == Some(entity_id) + && state + .get("item_count") + .and_then(serde_json::Value::as_u64) + .is_some() + && object_values_match(state.get("counters"), |value| value.as_u64().is_some()) + && object_values_match(state.get("booleans"), |value| value.as_bool().is_some()) + && object_values_match(state.get("lists"), |value| { + value + .as_array() + .is_some_and(|items| items.iter().all(|item| item.as_str().is_some())) + }) + && fields_match + && state + .get("events") + .and_then(serde_json::Value::as_array) + .is_some_and(Vec::is_empty) + && state + .get("total_event_count") + .and_then(serde_json::Value::as_u64) + .is_some() + && state + .get("events_since_snapshot") + .and_then(serde_json::Value::as_u64) + == Some(0) + && state + .get("last_snapshot_sequence_nr") + .and_then(serde_json::Value::as_u64) + == Some(0) + && state.get("sequence_nr").and_then(serde_json::Value::as_u64) == Some(0) + && processed_keys_match +} + +/// Return whether an envelope is the valid materialization control record for +/// the specified stream identity. +pub fn is_state_materialization_event_for( + envelope: &PersistenceEnvelope, + entity_type: &str, + entity_id: &str, +) -> bool { + is_state_materialization_payload_for( + &envelope.event_type, + &envelope.payload, + entity_type, + entity_id, + ) +} diff --git a/crates/temper-runtime/src/persistence/mod.rs b/crates/temper-runtime/src/persistence/mod.rs index 80c236adc..52e400e12 100644 --- a/crates/temper-runtime/src/persistence/mod.rs +++ b/crates/temper-runtime/src/persistence/mod.rs @@ -1,5 +1,28 @@ use serde::{Deserialize, Serialize}; +mod batch; +mod event_store_index; +mod event_store_journal; +mod index; +mod materialization; +mod types; +pub use batch::{PersistenceAppend, PersistenceAppendResult, PersistenceBatchIdempotency}; +pub use index::{ + EntityKeyLookup, EntityKeyRow, EntityVectorCandidate, EntityVectorRow, IndexReconciliation, + KeyContractActivation, KeyIndexBackfillFence, ProjectionSourceFence, SnapshotBackfillFence, + SnapshotSourceFence, decode_activated_key_contract, encode_activated_key_contract, pack_f32_le, + unpack_f32_le, +}; +pub use materialization::{ + STATE_MATERIALIZATION_EVENT_TYPE, STATE_MATERIALIZATION_IDEMPOTENCY_KEY_BUDGET, + STATE_MATERIALIZATION_PAYLOAD_BYTE_BUDGET, STATE_MATERIALIZATION_SCHEMA, + is_state_materialization_event_for, is_state_materialization_payload_for, + serialized_json_fits_byte_budget, +}; +pub use types::{ + JournalBoundary, KeyReconciliationEntity, PersistenceEnvelope, PersistenceError, storage_error, +}; + /// Event type used for the parent-journal record of a Composite action. /// /// Concrete sub-write events remain the state-changing events on their target @@ -15,6 +38,9 @@ pub struct CompositeEvent { pub parent_entity_id: String, pub parent_action: String, pub composite_idempotency_key: String, + /// Versioned digest of the complete normalized composite intent. + #[serde(default)] + pub intent_hash: String, pub sub_writes: Vec, } @@ -83,432 +109,9 @@ pub trait PersistentActor: Send + 'static { } } -/// A declared-key row to co-commit with an append (ADR-0153). The entity claims -/// `key_hash` for `key_name`; the store writes it into `entity_key_index` in the -/// same transaction as the journal append, giving the read plane an `O(log n)` -/// present/absent probe (the negative-existence access path, ARN-68). -#[derive(Debug, Clone)] -pub struct EntityKeyRow { - /// The declared key's identifier (the `[[key]]` block's `name`). - pub key_name: String, - /// The canonical, type-tagged hash of the key's values. - pub key_hash: String, -} - -/// A derived vector-index row to co-commit with an append (ADR-0155). Parsed from -/// the entity's post-transition state for one declared `[[vector]]` path: the -/// float vector and the model tag that partitions its space. Stores that maintain -/// `entity_vector_index` write one row per `(decl_name, model_tag, entity_id)`; the -/// blob is packed little-endian f32. Unlike a key row this has no uniqueness -/// constraint — it is derived, rebuildable ranking state. -#[derive(Debug, Clone, PartialEq)] -pub struct EntityVectorRow { - /// The declared vector path's identifier (the `[[vector]]` block's `name`). - pub decl_name: String, - /// The model tag that partitions this vector's space (only same-tag vectors - /// are ever compared). - pub model_tag: String, - /// The float vector, exactly `dims` long. - pub vector: Vec, -} - -/// Pack an `f32` slice to little-endian bytes — the `entity_vector_index` blob -/// encoding shared by every backend (ADR-0155). Kept here beside [`EntityVectorRow`] -/// so the stores and the kernel ranking agree on the byte layout. -pub fn pack_f32_le(vector: &[f32]) -> Vec { - let mut bytes = Vec::with_capacity(vector.len() * 4); - for value in vector { - bytes.extend_from_slice(&value.to_le_bytes()); - } - bytes -} - -/// Unpack little-endian bytes back to `f32`. `None` if the byte length is not a -/// multiple of 4, or if any component is not finite (both signal a corrupt blob), -/// so a bad row is skipped rather than panicking or feeding a `NaN`/`inf` into the -/// kNN ranking — where a `NaN` would sort ahead of every real score. -pub fn unpack_f32_le(bytes: &[u8]) -> Option> { - if !bytes.len().is_multiple_of(4) { - return None; - } - let mut out = Vec::with_capacity(bytes.len() / 4); - for chunk in bytes.chunks_exact(4) { - let value = f32::from_le_bytes([chunk[0], chunk[1], chunk[2], chunk[3]]); - if !value.is_finite() { - return None; - } - out.push(value); - } - Some(out) -} - -/// One candidate row returned from the vector index for a kNN read (ADR-0155): -/// an entity and its packed vector for one `(tenant, type, decl, model_tag)` -/// partition. The kernel — not the store — computes the metric over these in the -/// store-supplied (entity-id) order, so ranking is identical across backends. -#[derive(Debug, Clone, PartialEq)] -pub struct EntityVectorCandidate { - /// The entity holding this vector. - pub entity_id: String, - /// The float vector, exactly `dims` long. - pub vector: Vec, -} - /// Trait for the event store backend (implemented by temper-store-postgres). /// Uses desugared async-in-trait to enforce Send bounds on futures. pub trait EventStore: Send + Sync + 'static { - /// Append events to the journal. - fn append( - &self, - persistence_id: &str, - expected_sequence: u64, - events: &[PersistenceEnvelope], - ) -> impl std::future::Future> + Send; - - /// Append events and co-commit declared key-index rows (ADR-0153) in the - /// **same transaction** as the journal append. A thin forwarder to - /// [`EventStore::append_with_index_rows`] with no vector rows and no vector - /// reconcile, so callers that only maintain keys are unchanged. The co-commit - /// logic lives in `append_with_index_rows`, which query-plane backends override. - fn append_with_keys( - &self, - persistence_id: &str, - expected_sequence: u64, - events: &[PersistenceEnvelope], - key_rows: &[EntityKeyRow], - ) -> impl std::future::Future> + Send { - self.append_with_index_rows( - persistence_id, - expected_sequence, - events, - key_rows, - &[], - false, - ) - } - - /// Append events and co-commit BOTH declared key-index rows (ADR-0153) and - /// derived vector-index rows (ADR-0155) in the **same transaction** as the - /// journal append. This is the single co-commit entry point the entity actor - /// calls. The default ignores the index kinds and delegates to - /// [`EventStore::append`] — stores with a query plane that co-commit (postgres, - /// sim) override it; Turso also overrides it to maintain the vector index - /// write-behind (event first, index follows). When `reconcile_vectors` is true - /// (the entity's type declares ≥1 `[[vector]]` path) the store first DELETES all - /// of the entity's vector rows, then inserts `vector_rows` — so a delete - /// transition or a cleared vector/model property purges the stale rows instead of - /// leaving them to be ranked forever. The sequence and atomicity contract is - /// identical to `append`. - fn append_with_index_rows( - &self, - persistence_id: &str, - expected_sequence: u64, - events: &[PersistenceEnvelope], - key_rows: &[EntityKeyRow], - vector_rows: &[EntityVectorRow], - reconcile_vectors: bool, - ) -> impl std::future::Future> + Send { - let _ = (key_rows, vector_rows, reconcile_vectors); - self.append(persistence_id, expected_sequence, events) - } - - /// Reconcile the derived vector-index rows for an **existing** entity to exactly - /// `vector_rows` (ADR-0155), without appending a journal event: DELETE every - /// existing row for `(tenant, entity_type, entity_id)`, then INSERT `vector_rows`. - /// Idempotent, and an empty `vector_rows` PURGES the entity (used to clean up a - /// deleted or un-embedded entity). Used by the backfill and by the Turso - /// write-behind path. The default is a no-op (non-indexing backends); query-plane - /// stores implement it. - fn backfill_entity_vectors( - &self, - tenant: &str, - entity_type: &str, - entity_id: &str, - vector_rows: &[EntityVectorRow], - ) -> impl std::future::Future> + Send { - let _ = (tenant, entity_type, entity_id, vector_rows); - async { Ok(()) } - } - - /// The candidate `(entity_id, vector)` rows for one vector-index partition - /// `(tenant, entity_type, decl_name, model_tag)`, in **deterministic entity-id - /// order** (ADR-0155), capped at `limit` rows. The kernel ranks these; the store - /// only supplies the packed vectors, and applies `LIMIT` so an over-budget - /// partition is detected (caller passes `budget + 1`) without loading the whole - /// partition into memory. Default empty (non-indexing backends have no index). - fn vector_candidates( - &self, - tenant: &str, - entity_type: &str, - decl_name: &str, - model_tag: &str, - limit: usize, - ) -> impl std::future::Future, PersistenceError>> + Send - { - let _ = (tenant, entity_type, decl_name, model_tag, limit); - async { Ok(Vec::new()) } - } - - /// Record that `entity_vector_index` is **complete** for `(tenant, entity_type)` - /// — every existing entity has had its declared vectors indexed by the backfill - /// (ADR-0155 watermark, mirroring `mark_key_index_backfilled`). `vector_set` is - /// the sorted, comma-joined declared vector-path NAMES the backfill covered, so a - /// later declaration of an ADDITIONAL path is detected as a set change and the - /// type is re-indexed. Idempotent. Default no-op. - fn mark_vector_index_backfilled( - &self, - tenant: &str, - entity_type: &str, - vector_set: &str, - ) -> impl std::future::Future> + Send { - let _ = (tenant, entity_type, vector_set); - async { Ok(()) } - } - - /// The `(entity_type, vector_set)` watermarks for `tenant` — each type whose - /// `entity_vector_index` backfill is complete, paired with the covered path set. - /// Default empty (no backend authority). Mirrors `key_index_backfilled_types`. - fn vector_index_backfilled_types( - &self, - tenant: &str, - ) -> impl std::future::Future, PersistenceError>> + Send - { - let _ = tenant; - async { Ok(Vec::new()) } - } - - /// The `entity_id`s that already have at least one `entity_vector_index` row for - /// `(tenant, entity_type)`. Lets the vector backfill **resume** cheaply, skipping - /// already-indexed entities. Default empty (no resumption). Mirrors - /// `keyed_entity_ids_for_type`. - fn vectored_entity_ids_for_type( - &self, - tenant: &str, - entity_type: &str, - ) -> impl std::future::Future, PersistenceError>> + Send { - let _ = (tenant, entity_type); - async { Ok(Vec::new()) } - } - - /// Backfill declared key-index rows for an **existing** entity (ADR-0153), - /// without appending a journal event. Idempotent: re-running yields the same - /// rows. Used to populate `entity_key_index` for entities written before the - /// declared key existed, so a keyed read can authoritatively prove absence - /// (the per-tenant backfill watermark gates #324's retirement). The default - /// is a no-op (non-indexing backends); query-plane stores upsert the rows. - fn backfill_entity_keys( - &self, - tenant: &str, - entity_type: &str, - entity_id: &str, - key_rows: &[EntityKeyRow], - ) -> impl std::future::Future> + Send { - let _ = (tenant, entity_type, entity_id, key_rows); - async { Ok(()) } - } - - /// Resolve an entity by a declared key (ADR-0153): the `entity_id` currently - /// holding `(key_name, key_hash)`, or `None` if absent. This is the - /// negative-existence access path — present *and* absent in one `O(log n)` - /// probe, no scan. Default returns `None` (non-indexing backends); the - /// query-plane stores override it against `entity_key_index`. - fn lookup_by_key( - &self, - tenant: &str, - entity_type: &str, - key_name: &str, - key_hash: &str, - ) -> impl std::future::Future, PersistenceError>> + Send { - let _ = (tenant, entity_type, key_name, key_hash); - async { Ok(None) } - } - - /// Record that `entity_key_index` is **complete** for `(tenant, entity_type)` - /// — every existing entity of that type has been keyed by the backfill - /// (ADR-0153 watermark). Once set, a keyed read MISS is authoritative absence, - /// which retires the full-type reconcile scan (#324) for that type: the read - /// plane can answer "not found" without scanning. Idempotent. - /// - /// **Soundness invariant — only override this on a backend that co-commits key - /// rows on EVERY write** (i.e. overrides [`EventStore::append_with_keys`]). The - /// watermark asserts the index is complete *and stays complete*; a backend that - /// backfills but does not maintain keys live (e.g. Turso, which does not - /// co-commit) would let a later write go unkeyed, and a keyed miss for that - /// present entity would then read as authoritative absence — a silent - /// correctness bug. Such backends MUST keep the default no-op so they never - /// become authoritative (their keyed misses fall back to the scan — correct, - /// just not bounded). Postgres co-commits and overrides this; the sim store does - /// too for DST. The default is a no-op. - /// - /// `key_set` is the sorted, comma-joined declared key NAMES the backfill just - /// covered. It is recorded so a later declaration of an ADDITIONAL key is detected - /// as a key-set change (the recorded set no longer equals the current one) and the - /// type is re-keyed, instead of being wrongly treated as already complete. - fn mark_key_index_backfilled( - &self, - tenant: &str, - entity_type: &str, - key_set: &str, - ) -> impl std::future::Future> + Send { - let _ = (tenant, entity_type, key_set); - async { Ok(()) } - } - - /// The `(entity_type, key_set)` watermarks for `tenant` — each type whose - /// `entity_key_index` backfill is complete, paired with the sorted comma-joined - /// declared key names it covered. The read plane caches these so a keyed miss on a - /// type resolves to authoritative absence ONLY when the covered key-set still equals - /// the currently-declared one. Default empty (no backend authority → scan-safe). - fn key_index_backfilled_types( - &self, - tenant: &str, - ) -> impl std::future::Future, PersistenceError>> + Send - { - let _ = tenant; - async { Ok(Vec::new()) } - } - - /// The `entity_id`s that already have at least one `entity_key_index` row for - /// `(tenant, entity_type)`. Lets the backfill **resume** cheaply: it skips - /// already-keyed entities (the expensive part is loading each entity's state), - /// so a re-run after a partial pass only processes the remainder instead of - /// re-loading all N. Default empty (no resumption — a backend without the index - /// re-processes everything, which is correct, just not incremental). - fn keyed_entity_ids_for_type( - &self, - tenant: &str, - entity_type: &str, - ) -> impl std::future::Future, PersistenceError>> + Send { - let _ = (tenant, entity_type); - async { Ok(Vec::new()) } - } - - /// Atomically append events to multiple journals. - /// - /// Backends must either commit every append in `appends`, or commit none. - /// This is the storage primitive composite actions need before they can - /// persist cross-actor sub-writes as one physical unit. - fn append_batch( - &self, - appends: &[PersistenceAppend], - ) -> impl std::future::Future, PersistenceError>> + Send; - - /// Read events from the journal, starting after the given sequence number. - fn read_events( - &self, - persistence_id: &str, - from_sequence: u64, - ) -> impl std::future::Future, PersistenceError>> + Send; - - /// Save a state snapshot. - fn save_snapshot( - &self, - persistence_id: &str, - sequence_nr: u64, - snapshot: &[u8], - ) -> impl std::future::Future> + Send; - - /// Load the latest snapshot. - fn load_snapshot( - &self, - persistence_id: &str, - ) -> impl std::future::Future)>, PersistenceError>> + Send; - - /// List all distinct `(entity_type, entity_id)` pairs for a tenant. - fn list_entity_ids( - &self, - tenant: &str, - ) -> impl std::future::Future, PersistenceError>> + Send; - - /// List distinct entity IDs for one `(tenant, entity_type)` pair. - fn list_entity_ids_by_type( - &self, - tenant: &str, - entity_type: &str, - ) -> impl std::future::Future, PersistenceError>> + Send; - - /// List at most `limit` authoritative `(entity_type, entity_id)` pairs for - /// a tenant, optionally scoped to one entity type. - /// - /// Storage backends should override this to apply the bound inside the - /// backing query. The default is intended for small in-memory/test stores. - fn list_entity_ids_limited( - &self, - tenant: &str, - entity_type: Option<&str>, - limit: usize, - ) -> impl std::future::Future, PersistenceError>> + Send - { - async move { - if limit == 0 { - return Ok(Vec::new()); - } - let mut entities = if let Some(entity_type) = entity_type { - self.list_entity_ids_by_type(tenant, entity_type) - .await? - .into_iter() - .map(|entity_id| (entity_type.to_string(), entity_id)) - .collect::>() - } else { - self.list_entity_ids(tenant).await? - }; - entities.sort(); - entities.truncate(limit); - Ok(entities) - } - } -} - -/// A persisted event with metadata. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PersistenceEnvelope { - /// Monotonic sequence number within the entity's journal. - pub sequence_nr: u64, - /// Fully qualified event type name. - pub event_type: String, - /// Serialized event payload. - pub payload: serde_json::Value, - /// Event metadata (causation, correlation, timestamp). - pub metadata: EventMetadata, -} - -/// One stream append inside an atomic multi-journal append. -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct PersistenceAppend { - /// Persistence ID in the form `{tenant}:{entity_type}:{entity_id}`. - pub persistence_id: String, - /// Optimistic-concurrency sequence expected before this append. - pub expected_sequence: u64, - /// Events to append to this journal. - pub events: Vec, -} - -/// New sequence number for one stream after an atomic batch append. -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -pub struct PersistenceAppendResult { - /// Persistence ID that was appended. - pub persistence_id: String, - /// New highest sequence number for this journal. - pub sequence_nr: u64, -} - -/// Errors that can occur during event persistence operations. -#[derive(Debug, thiserror::Error)] -pub enum PersistenceError { - /// Optimistic concurrency check failed (another writer appended first). - #[error("optimistic concurrency violation: expected sequence {expected}, got {actual}")] - ConcurrencyViolation { expected: u64, actual: u64 }, - - /// Event serialization or deserialization failed. - #[error("serialization error: {0}")] - Serialization(String), - - /// Underlying storage backend returned an error. - #[error("storage error: {0}")] - Storage(String), -} - -/// Convert backend-specific errors into [`PersistenceError::Storage`]. -pub fn storage_error(err: impl std::fmt::Display) -> PersistenceError { - PersistenceError::Storage(err.to_string()) + event_store_index::event_store_index_methods!(); + event_store_journal::event_store_journal_methods!(); } diff --git a/crates/temper-runtime/src/persistence/types.rs b/crates/temper-runtime/src/persistence/types.rs new file mode 100644 index 000000000..398554a58 --- /dev/null +++ b/crates/temper-runtime/src/persistence/types.rs @@ -0,0 +1,154 @@ +//! Persisted envelope and backend error types. + +use serde::{Deserialize, Serialize}; + +use super::EventMetadata; + +/// A persisted event with metadata. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct PersistenceEnvelope { + /// Monotonic sequence number within the entity's journal. + pub sequence_nr: u64, + /// Fully qualified event type name. + pub event_type: String, + /// Serialized event payload. + pub payload: serde_json::Value, + /// Event metadata (causation, correlation, timestamp). + pub metadata: EventMetadata, +} + +/// Exact durable journal high-water and terminal lifecycle boundary for one stream. +/// +/// Recovery uses both values together: `latest_sequence` proves that a replay did +/// not stop at a fault-truncated prefix, while `first_terminal_sequence` prevents a +/// newer derived snapshot from reviving a stream that was already deleted. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct JournalBoundary { + /// Highest durable event sequence, or zero when the stream has no events. + pub latest_sequence: u64, + /// First event that durably transitioned the entity to `Deleted`, if any. + pub first_terminal_sequence: Option, +} + +/// One bounded declared-key repair candidate and its durable liveness at the +/// captured reconciliation revision. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct KeyReconciliationEntity { + /// Stable entity identifier used as the pagination cursor. + pub entity_id: String, + /// Whether a non-terminal authoritative source currently represents the entity. + pub is_live: bool, +} + +impl PersistenceEnvelope { + /// Whether this durable event makes the entity terminally deleted. + /// + /// Legacy writers used the canonical `Deleted` event/action name. Normal specs + /// may instead name the action `Delete` (or anything else) while persisting the + /// lifecycle boundary in `to_status`. + pub fn transitions_to_deleted(&self) -> bool { + match self.payload.get("to_status") { + Some(serde_json::Value::String(status)) => status == "Deleted", + // Structured lifecycle metadata is authoritative even when malformed: + // legacy name inference is only valid when the field is absent. + Some(_) => false, + None => { + self.event_type == "Deleted" + || self + .payload + .get("action") + .and_then(serde_json::Value::as_str) + == Some("Deleted") + } + } + } +} + +/// Errors that can occur during event persistence operations. +#[derive(Debug, thiserror::Error)] +pub enum PersistenceError { + /// Optimistic concurrency check failed (another writer appended first). + #[error("optimistic concurrency violation: expected sequence {expected}, got {actual}")] + ConcurrencyViolation { expected: u64, actual: u64 }, + /// A key-index repair was derived under a type contract that is no longer current. + #[error( + "key contract changed: expected '{expected_signature}' at revision {expected_revision}, got {actual_signature:?} at revision {actual_revision}" + )] + KeyContractChanged { + /// Signature used to derive the attempted repair rows. + expected_signature: String, + /// Contract revision captured before replay began. + expected_revision: u64, + /// Signature current when the repair acquired the type fence. + actual_signature: Option, + /// Contract revision current when the repair acquired the type fence. + actual_revision: u64, + }, + /// A delayed writer used a key signature that is no longer the activated + /// spec contract for this entity type. + #[error( + "key contract is not active: activated '{activated_signature}', attempted '{attempted_signature}'" + )] + KeyContractNotActive { + /// Signature established by the latest spec activation. + activated_signature: String, + /// Signature carried by the stale durable writer. + attempted_signature: String, + }, + /// A writer was derived before the latest activation of an otherwise + /// identical key signature (the A -> none -> A ABA case). + #[error( + "key contract activation is stale: active epoch {activated_epoch}, attempted {attempted_epoch:?}" + )] + KeyContractActivationStale { + /// Monotonic epoch established by the latest spec activation. + activated_epoch: u64, + /// Epoch captured by the writer's transition table. + attempted_epoch: Option, + }, + /// The activated contract has not yet completed a source-fenced ownership + /// rebuild. Live writes stay closed until its coverage publication succeeds. + #[error( + "key contract activation is not ready: active epoch {activated_epoch}, signature '{activated_signature}'" + )] + KeyContractActivationNotReady { + /// Monotonic epoch waiting for its ownership rebuild to publish. + activated_epoch: u64, + /// Activated declared-key signature awaiting coverage. + activated_signature: String, + }, + /// A key-index repair's entity classification became stale before mutation. + #[error( + "entity liveness changed during key repair: expected live={expected_live}, got live={actual_live}" + )] + EntityLivenessChanged { + /// Liveness observed by the repair pass before replay. + expected_live: bool, + /// Liveness observed under the store's stream lock. + actual_live: bool, + }, + /// A key-index repair's exact journal source boundary changed before mutation. + #[error( + "journal boundary changed during key repair: expected sequence {expected}, got {actual}" + )] + JournalBoundaryChanged { + /// Journal high-water captured before state reconstruction. + expected: u64, + /// Journal high-water observed under the store's stream lock. + actual: u64, + }, + /// Exact snapshot source changed before a derived write acquired its stream fence. + #[error("snapshot generation changed before durable write")] + SnapshotGenerationChanged, + /// Event serialization or deserialization failed. + #[error("serialization error: {0}")] + Serialization(String), + /// Underlying storage backend returned an error. + #[error("storage error: {0}")] + Storage(String), +} + +/// Convert backend-specific errors into [`PersistenceError::Storage`]. +pub fn storage_error(err: impl std::fmt::Display) -> PersistenceError { + PersistenceError::Storage(err.to_string()) +} diff --git a/crates/temper-runtime/tests/persistence_tombstone.rs b/crates/temper-runtime/tests/persistence_tombstone.rs new file mode 100644 index 000000000..15ac6268e --- /dev/null +++ b/crates/temper-runtime/tests/persistence_tombstone.rs @@ -0,0 +1,40 @@ +use temper_runtime::persistence::{EventMetadata, PersistenceEnvelope}; + +fn envelope(event_type: &str, payload: serde_json::Value) -> PersistenceEnvelope { + PersistenceEnvelope { + sequence_nr: 1, + event_type: event_type.to_string(), + payload, + metadata: EventMetadata { + event_id: uuid::Uuid::nil(), + causation_id: uuid::Uuid::nil(), + correlation_id: uuid::Uuid::nil(), + timestamp: chrono::DateTime::UNIX_EPOCH, + actor_id: "default:Doc:doc-1".to_string(), + }, + } +} + +#[test] +fn explicit_live_target_status_outranks_legacy_deleted_names() { + let explicitly_live = envelope( + "Deleted", + serde_json::json!({ + "action": "Deleted", + "from_status": "Ready", + "to_status": "Ready" + }), + ); + + assert!( + !explicitly_live.transitions_to_deleted(), + "structured lifecycle metadata must outrank legacy event/action names" + ); +} + +#[test] +fn legacy_deleted_name_remains_a_tombstone_without_target_status() { + let legacy = envelope("Deleted", serde_json::json!({"action": "Deleted"})); + + assert!(legacy.transitions_to_deleted()); +} diff --git a/crates/temper-server/src/api/decisions.rs b/crates/temper-server/src/api/decisions.rs index 113deef8e..3f825ed52 100644 --- a/crates/temper-server/src/api/decisions.rs +++ b/crates/temper-server/src/api/decisions.rs @@ -19,7 +19,7 @@ use tracing::instrument; use temper_runtime::tenant::TenantId; use super::{PolicyAuthed, decisions_access, empty_decision_list, format_decision_list}; -use crate::authz::{persist_and_activate_policy, require_observe_auth}; +use crate::authz::require_observe_auth; use crate::request_context::AgentContext; use crate::state::{DecisionStatus, PendingDecision, ServerState}; @@ -72,7 +72,7 @@ pub(crate) async fn handle_list_decisions( pub(crate) async fn handle_approve_decision( State(state): State, Path((tenant, id)): Path<(String, String)>, - _auth: PolicyAuthed, + mut auth: PolicyAuthed, axum::Json(body): axum::Json, ) -> impl IntoResponse { let scope = body.scope; @@ -129,35 +129,21 @@ pub(crate) async fn handle_approve_decision( let generated_policy = decision.generate_policy_from_matrix(&scope); let evolution_record_id = decision.evolution_record_id.clone(); - // Validate the generated policy combined with existing enabled policies. - let prospective = { - let policies = state.tenant_policies.read().unwrap(); // ci-ok: infallible lock - let existing = policies.get(&tenant).cloned().unwrap_or_default(); - if existing.is_empty() { - generated_policy.clone() - } else { - format!("{existing}\n{generated_policy}") - } - }; - if let Err(resp) = super::validate_and_reload_policies(&state, &tenant, &prospective) { - return resp; - } - - // Persist the individual policy to the granular `policies` table. let decided_by_ref = body.decided_by.as_deref().unwrap_or("unknown"); - persist_and_activate_policy( + let expected_generation = auth.release_for_publication(); + let auth_headers = auth.headers().clone(); + if let Err(response) = super::policies::publish_policy_upsert( &state, &tenant, &format!("decision:{id}"), &generated_policy, decided_by_ref, + Some(expected_generation), + Some(&auth_headers), ) - .await; - - // Update in-memory map to reflect the new policy. + .await { - let mut policies = state.tenant_policies.write().unwrap(); // ci-ok: infallible lock - policies.insert(tenant.clone(), prospective); + return response; } // Mark decision approved only after policy reload succeeds. @@ -170,7 +156,12 @@ pub(crate) async fn handle_approve_decision( // Persist updated decision synchronously. if let Err(e) = state.persist_pending_decision(&approved_decision).await { - tracing::warn!(id = %id, error = %e, "failed to persist approved decision"); + tracing::error!(id = %id, error = %e, "failed to persist approved decision"); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to persist approved decision: {e}"), + ) + .into_response(); } let _ = state .observe_refresh_tx @@ -235,6 +226,7 @@ pub(crate) async fn handle_approve_decision( "decided_by": decided_by, "scope": "narrow", "generated_policy": generated_policy, + "policy_already_published": true, }), &AgentContext::for_service("platform-dispatch"), ) @@ -322,7 +314,12 @@ pub(crate) async fn handle_deny_decision( // Persist updated decision synchronously. if let Err(e) = state.persist_pending_decision(&denied_decision).await { - tracing::warn!(error = %e, "failed to persist denied decision"); + tracing::error!(id = %id, error = %e, "failed to persist denied decision"); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to persist denied decision: {e}"), + ) + .into_response(); } let _ = state .observe_refresh_tx diff --git a/crates/temper-server/src/api/mod.rs b/crates/temper-server/src/api/mod.rs index 6f9499e61..66e2f47e1 100644 --- a/crates/temper-server/src/api/mod.rs +++ b/crates/temper-server/src/api/mod.rs @@ -234,7 +234,42 @@ pub(crate) async fn require_policy_auth( /// produces (403 + `AuthorizationDenied` JSON including the decision id). /// The tenant is read from the request parts by name, so handlers keep their /// own `Path` / `Path<(String, String)>` extractors untouched. -pub(crate) struct PolicyAuthed; +pub(crate) struct PolicyAuthed { + headers: HeaderMap, + generation: Option>, + captured_generation: u64, +} + +impl PolicyAuthed { + pub(crate) fn headers(&self) -> &HeaderMap { + &self.headers + } + + /// Release the stable authorization generation before a policy publisher + /// takes the write side of the same tenant barrier. + pub(crate) fn release_for_publication(&mut self) -> u64 { + self.generation.take(); + self.captured_generation + } +} + +fn is_policy_publication_retry(parts: &Parts) -> bool { + let path = parts.uri.path(); + let tenant_policy = path.contains("/tenants/") + && path.contains("/policies") + && matches!( + parts.method, + axum::http::Method::POST + | axum::http::Method::PUT + | axum::http::Method::PATCH + | axum::http::Method::DELETE + ); + let decision_approval = path.contains("/tenants/") + && path.contains("/decisions/") + && path.ends_with("/approve") + && parts.method == axum::http::Method::POST; + tenant_policy || decision_approval +} impl FromRequestParts for PolicyAuthed { type Rejection = axum::response::Response; @@ -252,9 +287,61 @@ impl FromRequestParts for PolicyAuthed { // {tenant} path parameter; reaching this branch is a routing bug. return Err(StatusCode::INTERNAL_SERVER_ERROR.into_response()); }; + let tenant_id = temper_runtime::tenant::TenantId::new(tenant); + let generation = if state.spec_publication_gated(&tenant_id) { + if !is_policy_publication_retry(parts) { + return Err(( + StatusCode::SERVICE_UNAVAILABLE, + "Tenant runtime generation has an unresolved publication", + ) + .into_response()); + } + let guard = state + .try_begin_tenant_request(&tenant_id) + .await + .ok_or_else(|| { + ( + StatusCode::SERVICE_UNAVAILABLE, + "Tenant runtime generation is busy", + ) + .into_response() + })?; + if !state.spec_publication_gated(&tenant_id) { + return Err(( + StatusCode::SERVICE_UNAVAILABLE, + "Tenant runtime generation changed during retry admission", + ) + .into_response()); + } + Some(guard) + } else { + let guard = state + .try_begin_tenant_request(&tenant_id) + .await + .ok_or_else(|| { + ( + StatusCode::SERVICE_UNAVAILABLE, + "Tenant runtime generation is busy", + ) + .into_response() + })?; + if state.spec_publication_gated(&tenant_id) { + return Err(( + StatusCode::SERVICE_UNAVAILABLE, + "Tenant runtime generation has an unresolved publication", + ) + .into_response()); + } + Some(guard) + }; + let captured_generation = state.tenant_generation_version(&tenant_id); match require_policy_auth(state, &parts.headers, tenant).await { Some(resp) => Err(resp), - None => Ok(Self), + None => Ok(Self { + headers: parts.headers.clone(), + generation, + captured_generation, + }), } } } diff --git a/crates/temper-server/src/api/policies.rs b/crates/temper-server/src/api/policies.rs index 1bbd8a165..98983121f 100644 --- a/crates/temper-server/src/api/policies.rs +++ b/crates/temper-server/src/api/policies.rs @@ -4,42 +4,29 @@ //! incremental rule addition, individual policy listing/toggling/editing/deletion, //! and cross-tenant policy views. +mod listing; +mod publication; + +pub(crate) use listing::{handle_list_all_policies, handle_list_policies}; +use publication::{ + PolicyUpsert, activate_policy_generation, arm_policy_generation, + begin_durable_policy_generation, begin_policy_generation_mutation, + begin_policy_generation_read, policy_generation_intent, policy_generation_writes, + publish_memory_policy_generation, publish_policy_upsert_mode, validate_policy_generation, +}; +pub(super) use publication::{publish_policy_replace_all, publish_policy_upsert}; + use axum::extract::{Path, State}; -use axum::http::{HeaderMap, StatusCode}; +use axum::http::StatusCode; use axum::response::IntoResponse; use tracing::instrument; use super::PolicyAuthed; -use crate::authz::{load_and_activate_tenant_policies, persist_and_activate_policy}; +use crate::authz::policy_persistence::persist_complete_policy_generation; use crate::state::ServerState; -use crate::storage::PolicyStoreRow; -/// Derive a human-readable source label from a `policy_id`. -fn policy_source(policy_id: &str) -> &'static str { - if policy_id.starts_with("os-app:") { - "os-app" - } else if policy_id.starts_with("decision:") { - "decision" - } else if policy_id == "migrated-legacy" { - "migrated-legacy" - } else { - "manual" - } -} - -/// Serialize a [`PolicyRow`] to a JSON value for API responses. -fn policy_row_to_json(row: &PolicyStoreRow) -> serde_json::Value { - serde_json::json!({ - "tenant": row.tenant, - "policy_id": row.policy_id, - "cedar_text": row.cedar_text, - "enabled": row.enabled, - "policy_hash": row.policy_hash, - "created_at": row.created_at, - "created_by": row.created_by, - "source": policy_source(&row.policy_id), - }) -} +#[cfg(test)] +mod tests; // --------------------------------------------------------------------------- // Existing endpoints (unchanged interface, kept for backward compatibility) @@ -70,7 +57,7 @@ pub(crate) async fn handle_get_policies( pub(crate) async fn handle_put_policies( State(state): State, Path(tenant): Path, - _auth: PolicyAuthed, + mut auth: PolicyAuthed, body: axum::body::Bytes, ) -> impl IntoResponse { let body_json: serde_json::Value = match serde_json::from_slice(&body) { @@ -93,21 +80,44 @@ pub(crate) async fn handle_put_policies( } }; - if let Err(resp) = super::validate_and_reload_policies(&state, &tenant, &policy_text) { - return resp; - } - - { - let mut policies = state.tenant_policies.write().unwrap(); // ci-ok: infallible lock - policies.insert(tenant.clone(), policy_text.clone()); + let expected_generation = auth.release_for_publication(); + let auth_headers = auth.headers().clone(); + if state.policy_store().is_some() { + if let Err(response) = publish_policy_replace_all( + &state, + &tenant, + &policy_text, + "api", + Some(expected_generation), + Some(&auth_headers), + ) + .await + { + return response; + } + } else { + let mut generation_writer = match begin_policy_generation_mutation( + &state, + &tenant, + Some(expected_generation), + Some(&auth_headers), + ) + .await + { + Ok(guard) => guard, + Err(response) => return response, + }; + if let Err(response) = publish_memory_policy_generation( + &state, + &tenant, + &policy_text, + "memory-policy-replace-v1", + &mut generation_writer, + ) { + return response; + } } - persist_and_activate_policy(&state, &tenant, "primary", &policy_text, "api").await; - - let _ = state - .observe_refresh_tx - .send(crate::state::ObserveRefreshHint::Policies); - ( StatusCode::OK, axum::Json(serde_json::json!({"tenant": tenant, "status": "loaded"})), @@ -122,7 +132,7 @@ pub(crate) async fn handle_put_policies( pub(crate) async fn handle_add_policy_rule( State(state): State, Path(tenant): Path, - _auth: PolicyAuthed, + mut auth: PolicyAuthed, body: axum::body::Bytes, ) -> impl IntoResponse { let body_json: serde_json::Value = match serde_json::from_slice(&body) { @@ -145,140 +155,60 @@ pub(crate) async fn handle_add_policy_rule( } }; - let new_tenant_text = { - let policies = state.tenant_policies.read().unwrap(); // ci-ok: infallible lock - let existing = policies.get(&tenant).cloned().unwrap_or_default(); - if existing.is_empty() { - rule.clone() - } else { - format!("{existing}\n{rule}") - } - }; - - if let Err(resp) = super::validate_and_reload_policies(&state, &tenant, &new_tenant_text) { - return resp; - } - - { - let mut policies = state.tenant_policies.write().unwrap(); // ci-ok: infallible lock - policies.insert(tenant.clone(), new_tenant_text.clone()); - } - - persist_and_activate_policy(&state, &tenant, "primary", &new_tenant_text, "api").await; - - let _ = state - .observe_refresh_tx - .send(crate::state::ObserveRefreshHint::Policies); - - ( - StatusCode::OK, - axum::Json(serde_json::json!({"tenant": tenant, "status": "rule_added"})), - ) - .into_response() -} - -// --------------------------------------------------------------------------- -// New individual policy management endpoints (Phase 1) -// --------------------------------------------------------------------------- - -/// GET /api/tenants/{tenant}/policies/list — list individual policy entries. -/// -/// Returns structured JSON with per-policy details (id, cedar_text, enabled, etc.). -/// Cedar-gated: requires `manage_policies` action on `PolicySet` resource. -#[instrument(skip_all, fields(tenant, otel.name = "GET /api/tenants/{tenant}/policies/list"))] -pub(crate) async fn handle_list_policies( - State(state): State, - Path(tenant): Path, - _auth: PolicyAuthed, -) -> impl IntoResponse { - let Some(store) = state.policy_store() else { - return ( - StatusCode::SERVICE_UNAVAILABLE, - "Persistence backend not configured", + let expected_generation = auth.release_for_publication(); + let auth_headers = auth.headers().clone(); + if state.policy_store().is_some() { + if let Err(response) = publish_policy_upsert_mode( + &state, + &tenant, + "primary", + PolicyUpsert::AppendRule(rule), + "api", + Some(expected_generation), + Some(&auth_headers), ) - .into_response(); - }; - - match store.load_policies_for_tenant(&tenant).await { - Ok(rows) => { - let enabled_count = rows.iter().filter(|r| r.enabled).count(); - let disabled_count = rows.len() - enabled_count; - let policies: Vec = rows.iter().map(policy_row_to_json).collect(); - ( - StatusCode::OK, - axum::Json(serde_json::json!({ - "tenant": tenant, - "policies": policies, - "total": rows.len(), - "enabled_count": enabled_count, - "disabled_count": disabled_count, - })), - ) - .into_response() - } - Err(e) => { - tracing::warn!(error = %e, tenant, "failed to list policies"); - ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to list policies: {e}"), - ) - .into_response() + .await + { + return response; } - } -} - -/// GET /api/policies — list policies across all tenants (admin only). -#[instrument(skip_all, fields(otel.name = "GET /api/policies"))] -pub(crate) async fn handle_list_all_policies( - State(state): State, - headers: HeaderMap, -) -> impl IntoResponse { - if let Err(status) = - crate::authz::require_observe_auth(&state, &headers, "manage_policies", "PolicySet") - { - return (status, "Authorization required for cross-tenant access").into_response(); - } - - let Some(store) = state.policy_store() else { - return ( - StatusCode::SERVICE_UNAVAILABLE, - "Persistence backend not configured", + } else { + let mut generation_writer = match begin_policy_generation_mutation( + &state, + &tenant, + Some(expected_generation), + Some(&auth_headers), ) - .into_response(); - }; - - let mut rows = match store.load_all_policies().await { - Ok(rows) => rows, - Err(e) => { - tracing::warn!(error = %e, "failed to list policies from durable store"); - return ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to list policies: {e}"), - ) - .into_response(); + .await + { + Ok(guard) => guard, + Err(response) => return response, + }; + let new_tenant_text = { + let policies = state + .tenant_policies + .read() + .expect("tenant policy lock poisoned"); + let existing = policies.get(&tenant).cloned().unwrap_or_default(); + if existing.is_empty() { + rule + } else { + format!("{existing}\n{rule}") + } + }; + if let Err(response) = publish_memory_policy_generation( + &state, + &tenant, + &new_tenant_text, + "memory-policy-append-v1", + &mut generation_writer, + ) { + return response; } - }; - - rows.sort_by(|a, b| { - a.tenant - .cmp(&b.tenant) - .then_with(|| a.policy_id.cmp(&b.policy_id)) - .then_with(|| a.created_at.cmp(&b.created_at)) - }); - - let total = rows.len(); - let mut by_tenant = std::collections::BTreeMap::new(); - for row in &rows { - *by_tenant.entry(row.tenant.clone()).or_insert(0usize) += 1; } - let policies: Vec = rows.iter().map(policy_row_to_json).collect(); + ( StatusCode::OK, - axum::Json(serde_json::json!({ - "policies": policies, - "total": total, - "by_tenant": by_tenant, - })), + axum::Json(serde_json::json!({"tenant": tenant, "status": "rule_added"})), ) .into_response() } @@ -291,7 +221,7 @@ pub(crate) async fn handle_list_all_policies( pub(crate) async fn handle_create_policy( State(state): State, Path(tenant): Path, - _auth: PolicyAuthed, + mut auth: PolicyAuthed, axum::Json(body): axum::Json, ) -> impl IntoResponse { let policy_id = match body.get("policy_id").and_then(|v| v.as_str()) { @@ -315,30 +245,61 @@ pub(crate) async fn handle_create_policy( } }; - // Validate: build prospective enabled policy text with the new entry added. - let prospective = - build_prospective_enabled_text(&state, &tenant, Some((&policy_id, &cedar_text))).await; - if let Err(resp) = super::validate_and_reload_policies(&state, &tenant, &prospective) { - return resp; - } - - // Persist the new policy entry. let created_by = body .get("created_by") .and_then(|v| v.as_str()) .unwrap_or("api"); - persist_and_activate_policy(&state, &tenant, &policy_id, &cedar_text, created_by).await; - - // Update in-memory map to match the prospective text. - { - let mut policies = state.tenant_policies.write().unwrap(); // ci-ok: infallible lock - policies.insert(tenant.clone(), prospective); + let expected_generation = auth.release_for_publication(); + let auth_headers = auth.headers().clone(); + if state.policy_store().is_some() { + if let Err(response) = publish_policy_upsert( + &state, + &tenant, + &policy_id, + &cedar_text, + created_by, + Some(expected_generation), + Some(&auth_headers), + ) + .await + { + return response; + } + } else { + let mut generation_writer = match begin_policy_generation_mutation( + &state, + &tenant, + Some(expected_generation), + Some(&auth_headers), + ) + .await + { + Ok(guard) => guard, + Err(response) => return response, + }; + let prospective = { + let policies = state + .tenant_policies + .read() + .expect("tenant policy lock poisoned"); + let existing = policies.get(&tenant).cloned().unwrap_or_default(); + if existing.is_empty() { + cedar_text.clone() + } else { + format!("{existing}\n{cedar_text}") + } + }; + if let Err(response) = publish_memory_policy_generation( + &state, + &tenant, + &prospective, + "memory-policy-create-v1", + &mut generation_writer, + ) { + return response; + } } - let _ = state - .observe_refresh_tx - .send(crate::state::ObserveRefreshHint::Policies); - ( StatusCode::CREATED, axum::Json(serde_json::json!({ @@ -359,17 +320,9 @@ pub(crate) async fn handle_create_policy( pub(crate) async fn handle_patch_policy( State(state): State, Path((tenant, policy_id)): Path<(String, String)>, - _auth: PolicyAuthed, + mut auth: PolicyAuthed, axum::Json(body): axum::Json, ) -> impl IntoResponse { - let Some(store) = state.policy_store() else { - return ( - StatusCode::SERVICE_UNAVAILABLE, - "Persistence backend not configured", - ) - .into_response(); - }; - let new_cedar_text = body.get("cedar_text").and_then(|v| v.as_str()); let new_enabled = body.get("enabled").and_then(|v| v.as_bool()); @@ -381,65 +334,76 @@ pub(crate) async fn handle_patch_policy( .into_response(); } - // If cedar_text is being changed, validate it first. + let expected_generation = auth.release_for_publication(); + let auth_headers = auth.headers().clone(); + let (mut generation_writer, _store, mut entries) = match begin_durable_policy_generation( + &state, + &tenant, + Some(expected_generation), + Some(&auth_headers), + ) + .await + { + Ok(generation) => generation, + Err(response) => return response, + }; + let Some(entry_index) = entries + .iter() + .position(|entry| entry.policy_id == policy_id) + else { + return (StatusCode::NOT_FOUND, "Policy not found").into_response(); + }; if let Some(cedar_text) = new_cedar_text { - // Validate by building prospective text for the tenant. - let prospective = build_prospective_enabled_text_with_override( - &state, - &tenant, - &policy_id, - cedar_text, - new_enabled, - ) - .await; - if let Err(resp) = super::validate_and_reload_policies(&state, &tenant, &prospective) { - return resp; - } - - let created_by = body - .get("created_by") - .and_then(|v| v.as_str()) - .unwrap_or("api"); - if let Err(e) = store - .update_policy_text(&tenant, &policy_id, cedar_text, created_by) - .await - { - tracing::warn!(error = %e, "failed to update policy text"); - return ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to update policy: {e}"), - ) - .into_response(); - } + entries[entry_index].cedar_text = cedar_text.to_string(); } - - // If enabled is being changed, toggle it. if let Some(enabled) = new_enabled { - match store - .toggle_policy_enabled(&tenant, &policy_id, enabled) - .await - { - Ok(false) => { - return (StatusCode::NOT_FOUND, "Policy not found").into_response(); - } - Err(e) => { - tracing::warn!(error = %e, "failed to toggle policy enabled"); - return ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to toggle policy: {e}"), - ) - .into_response(); - } - Ok(true) => {} - } + entries[entry_index].enabled = enabled; + } + if let Err(response) = validate_policy_generation(&tenant, &entries) { + return response; + } + let created_by = body + .get("created_by") + .and_then(|v| v.as_str()) + .unwrap_or("api"); + entries[entry_index].created_by = created_by.to_string(); + let enabled_component = new_enabled + .map(|enabled| enabled.to_string()) + .unwrap_or_default(); + let intent = policy_generation_intent( + "direct-policy-patch-v1", + &entries, + &[ + ("policy-id", policy_id.as_bytes()), + ("cedar-text", new_cedar_text.unwrap_or_default().as_bytes()), + ("enabled", enabled_component.as_bytes()), + ("created-by", created_by.as_bytes()), + ], + ); + if let Err(response) = arm_policy_generation(&state, &mut generation_writer, &tenant, &intent) { + return response; + } + if let Err(error) = persist_complete_policy_generation( + &state, + &tenant, + &policy_generation_writes(&entries), + &policy_id, + created_by, + ) + .await + { + tracing::warn!(%error, tenant, policy_id, "failed to replace policy generation"); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to update policy: {error}"), + ) + .into_response(); + } + if let Err(response) = + activate_policy_generation(&state, &tenant, &entries, &mut generation_writer) + { + return response; } - - // Reload tenant policies from durable storage to update in-memory state. - reload_tenant_from_store(&state, &tenant).await; - - let _ = state - .observe_refresh_tx - .send(crate::state::ObserveRefreshHint::Policies); ( StatusCode::OK, @@ -459,31 +423,54 @@ pub(crate) async fn handle_patch_policy( pub(crate) async fn handle_delete_policy_entry( State(state): State, Path((tenant, policy_id)): Path<(String, String)>, - _auth: PolicyAuthed, + mut auth: PolicyAuthed, ) -> impl IntoResponse { - let Some(store) = state.policy_store() else { - return ( - StatusCode::SERVICE_UNAVAILABLE, - "Persistence backend not configured", - ) - .into_response(); + let expected_generation = auth.release_for_publication(); + let auth_headers = auth.headers().clone(); + let (mut generation_writer, _store, mut entries) = match begin_durable_policy_generation( + &state, + &tenant, + Some(expected_generation), + Some(&auth_headers), + ) + .await + { + Ok(generation) => generation, + Err(response) => return response, }; - - if let Err(e) = store.delete_policy(&tenant, &policy_id).await { - tracing::warn!(error = %e, "failed to delete policy"); + entries.retain(|entry| entry.policy_id != policy_id); + if let Err(response) = validate_policy_generation(&tenant, &entries) { + return response; + } + let intent = policy_generation_intent( + "direct-policy-delete-v1", + &entries, + &[("policy-id", policy_id.as_bytes())], + ); + if let Err(response) = arm_policy_generation(&state, &mut generation_writer, &tenant, &intent) { + return response; + } + if let Err(error) = persist_complete_policy_generation( + &state, + &tenant, + &policy_generation_writes(&entries), + &policy_id, + "api", + ) + .await + { + tracing::warn!(%error, tenant, policy_id, "failed to delete policy generation"); return ( StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to delete policy: {e}"), + format!("Failed to delete policy: {error}"), ) .into_response(); } - - // Reload tenant policies from durable storage to update in-memory state. - reload_tenant_from_store(&state, &tenant).await; - - let _ = state - .observe_refresh_tx - .send(crate::state::ObserveRefreshHint::Policies); + if let Err(response) = + activate_policy_generation(&state, &tenant, &entries, &mut generation_writer) + { + return response; + } ( StatusCode::OK, @@ -495,78 +482,3 @@ pub(crate) async fn handle_delete_policy_entry( ) .into_response() } - -// --------------------------------------------------------------------------- -// Helpers -// --------------------------------------------------------------------------- - -/// Reload a tenant's in-memory policy state from durable storage. -/// -/// Reads all enabled policies, concatenates them, updates `tenant_policies`, -/// and reloads the Cedar engine. -async fn reload_tenant_from_store(state: &ServerState, tenant: &str) { - load_and_activate_tenant_policies(state, tenant).await; -} - -/// Build the prospective enabled policy text for a tenant, optionally including -/// a new policy entry that isn't persisted yet. -async fn build_prospective_enabled_text( - state: &ServerState, - tenant: &str, - additional: Option<(&str, &str)>, -) -> String { - let mut text = { - let policies = state.tenant_policies.read().unwrap(); // ci-ok: infallible lock - policies.get(tenant).cloned().unwrap_or_default() - }; - if let Some((_id, cedar_text)) = additional { - if !text.is_empty() { - text.push('\n'); - } - text.push_str(cedar_text); - } - text -} - -/// Build the prospective enabled policy text for a tenant, replacing one -/// specific policy entry's text and/or enabled state. -async fn build_prospective_enabled_text_with_override( - state: &ServerState, - tenant: &str, - override_policy_id: &str, - override_cedar_text: &str, - override_enabled: Option, -) -> String { - // Load all current policies from durable storage to get accurate per-entry data. - let rows = if let Some(store) = state.policy_store() { - store - .load_policies_for_tenant(tenant) - .await - .unwrap_or_default() - } else { - vec![] - }; - - let mut combined = String::new(); - for row in &rows { - let is_target = row.policy_id == override_policy_id; - let cedar_text = if is_target { - override_cedar_text - } else { - &row.cedar_text - }; - let enabled = if is_target { - override_enabled.unwrap_or(row.enabled) - } else { - row.enabled - }; - if !enabled { - continue; - } - if !combined.is_empty() { - combined.push('\n'); - } - combined.push_str(cedar_text); - } - combined -} diff --git a/crates/temper-server/src/api/policies/listing.rs b/crates/temper-server/src/api/policies/listing.rs new file mode 100644 index 000000000..19fc2ed37 --- /dev/null +++ b/crates/temper-server/src/api/policies/listing.rs @@ -0,0 +1,186 @@ +//! Stable-generation policy listing endpoints. + +use axum::extract::{Path, State}; +use axum::http::{HeaderMap, StatusCode}; +use axum::response::IntoResponse; +use tracing::instrument; + +use super::{PolicyAuthed, ServerState, begin_policy_generation_read}; +use crate::storage::PolicyStoreRow; + +fn policy_source(policy_id: &str) -> &'static str { + if policy_id.starts_with("os-app:") { + "os-app" + } else if policy_id.starts_with("decision:") { + "decision" + } else if policy_id == "migrated-legacy" { + "migrated-legacy" + } else { + "manual" + } +} + +fn policy_row_to_json(row: &PolicyStoreRow) -> serde_json::Value { + serde_json::json!({ + "tenant": row.tenant, + "policy_id": row.policy_id, + "cedar_text": row.cedar_text, + "enabled": row.enabled, + "policy_hash": row.policy_hash, + "created_at": row.created_at, + "created_by": row.created_by, + "source": policy_source(&row.policy_id), + }) +} + +/// GET /api/tenants/{tenant}/policies/list — list individual policy entries. +#[instrument(skip_all, fields(tenant, otel.name = "GET /api/tenants/{tenant}/policies/list"))] +pub(crate) async fn handle_list_policies( + State(state): State, + Path(tenant): Path, + _auth: PolicyAuthed, +) -> impl IntoResponse { + let Some(store) = state.policy_store() else { + return ( + StatusCode::SERVICE_UNAVAILABLE, + "Persistence backend not configured", + ) + .into_response(); + }; + + match store.load_policies_for_tenant(&tenant).await { + Ok(rows) => { + let enabled_count = rows.iter().filter(|row| row.enabled).count(); + let disabled_count = rows.len() - enabled_count; + let policies: Vec = rows.iter().map(policy_row_to_json).collect(); + ( + StatusCode::OK, + axum::Json(serde_json::json!({ + "tenant": tenant, + "policies": policies, + "total": rows.len(), + "enabled_count": enabled_count, + "disabled_count": disabled_count, + })), + ) + .into_response() + } + Err(error) => { + tracing::warn!(%error, tenant, "failed to list policies"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to list policies: {error}"), + ) + .into_response() + } + } +} + +/// GET /api/policies — list policies across all tenants (admin only). +#[instrument(skip_all, fields(otel.name = "GET /api/policies"))] +pub(crate) async fn handle_list_all_policies( + State(state): State, + headers: HeaderMap, +) -> impl IntoResponse { + if let Err(status) = + crate::authz::require_observe_auth(&state, &headers, "manage_policies", "PolicySet") + { + return (status, "Authorization required for cross-tenant access").into_response(); + } + + let Some(store) = state.policy_store() else { + return ( + StatusCode::SERVICE_UNAVAILABLE, + "Persistence backend not configured", + ) + .into_response(); + }; + + let discovery_rows = match store.load_all_policies().await { + Ok(rows) => rows, + Err(error) => { + tracing::warn!(%error, "failed to list policies from durable store"); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to list policies: {error}"), + ) + .into_response(); + } + }; + let mut tenants = discovery_rows + .iter() + .map(|row| row.tenant.clone()) + .collect::>(); + if let Ok(registry) = state.registry.read() { + tenants.extend( + registry + .tenant_ids() + .into_iter() + .map(|tenant| tenant.as_str().to_string()), + ); + } + let authorization_tenant = headers + .get("x-tenant-id") + .and_then(|value| value.to_str().ok()) + .filter(|tenant| !tenant.is_empty()) + .unwrap_or("system") + .to_string(); + tenants.insert(authorization_tenant); + let guarded_tenants = tenants.clone(); + let mut _generation_guards = Vec::with_capacity(tenants.len()); + for tenant in tenants { + match begin_policy_generation_read(&state, &tenant).await { + Ok(guard) => _generation_guards.push(guard), + Err(response) => return response, + } + } + if let Err(status) = + crate::authz::require_observe_auth(&state, &headers, "manage_policies", "PolicySet") + { + return (status, "Authorization required for cross-tenant access").into_response(); + } + let mut rows = match store.load_all_policies().await { + Ok(rows) => rows, + Err(error) => { + tracing::warn!(%error, "failed to list stable policies from durable store"); + return ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to list policies: {error}"), + ) + .into_response(); + } + }; + if rows + .iter() + .any(|row| !guarded_tenants.contains(&row.tenant)) + { + return ( + StatusCode::SERVICE_UNAVAILABLE, + "Policy tenant set changed while acquiring stable generations; retry", + ) + .into_response(); + } + + rows.sort_by(|a, b| { + a.tenant + .cmp(&b.tenant) + .then_with(|| a.policy_id.cmp(&b.policy_id)) + .then_with(|| a.created_at.cmp(&b.created_at)) + }); + + let total = rows.len(); + let mut by_tenant = std::collections::BTreeMap::new(); + for row in &rows { + *by_tenant.entry(row.tenant.clone()).or_insert(0usize) += 1; + } + let policies: Vec = rows.iter().map(policy_row_to_json).collect(); + ( + StatusCode::OK, + axum::Json(serde_json::json!({ + "policies": policies, + "total": total, + "by_tenant": by_tenant, + })), + ) + .into_response() +} diff --git a/crates/temper-server/src/api/policies/publication.rs b/crates/temper-server/src/api/policies/publication.rs new file mode 100644 index 000000000..08848f3e2 --- /dev/null +++ b/crates/temper-server/src/api/policies/publication.rs @@ -0,0 +1,446 @@ +//! Durable Cedar policy-generation publication helpers. + +use axum::http::{HeaderMap, StatusCode}; +use axum::response::IntoResponse; +use temper_runtime::tenant::TenantId; + +use crate::authz::policy_persistence::load_or_seed_policy_generation; +use crate::authz::policy_persistence::persist_complete_policy_generation; +use crate::state::{ServerState, SpecPublicationGuard}; +use crate::storage::{PolicyGenerationWrite, PolicyStore, PolicyStoreRow}; + +#[derive(Clone, Debug, PartialEq, Eq)] +pub(super) struct PolicyGenerationEntry { + pub(super) policy_id: String, + pub(super) cedar_text: String, + pub(super) enabled: bool, + pub(super) created_by: String, +} + +impl From<&PolicyStoreRow> for PolicyGenerationEntry { + fn from(row: &PolicyStoreRow) -> Self { + Self { + policy_id: row.policy_id.clone(), + cedar_text: row.cedar_text.clone(), + enabled: row.enabled, + created_by: row.created_by.clone(), + } + } +} + +pub(super) enum PolicyUpsert { + Replace(String), + AppendRule(String), +} + +pub(super) async fn begin_policy_generation_mutation( + state: &ServerState, + tenant: &str, + expected_generation: Option, + auth_headers: Option<&HeaderMap>, +) -> Result { + let tenant_id = TenantId::new(tenant); + let guard = match expected_generation { + Some(expected_generation) => { + state + .begin_spec_publication_after_drain(&tenant_id, expected_generation) + .await + } + None => state.begin_spec_publication(&tenant_id).await, + } + .map_err(|error| { + ( + StatusCode::SERVICE_UNAVAILABLE, + format!("Tenant runtime generation is busy: {error}"), + ) + .into_response() + })?; + if let Some(headers) = auth_headers + && let Some(response) = super::super::require_policy_auth(state, headers, tenant).await + { + return Err(response); + } + Ok(guard) +} + +pub(super) async fn begin_policy_generation_read( + state: &ServerState, + tenant: &str, +) -> Result, axum::response::Response> { + let tenant_id = TenantId::new(tenant); + if state.spec_publication_gated(&tenant_id) { + return Err(( + StatusCode::SERVICE_UNAVAILABLE, + "Tenant runtime generation has an unresolved publication", + ) + .into_response()); + } + let guard = state + .try_begin_tenant_request(&tenant_id) + .await + .ok_or_else(|| { + ( + StatusCode::SERVICE_UNAVAILABLE, + "Tenant runtime generation is busy", + ) + .into_response() + })?; + if state.spec_publication_gated(&tenant_id) { + return Err(( + StatusCode::SERVICE_UNAVAILABLE, + "Tenant runtime generation has an unresolved publication", + ) + .into_response()); + } + Ok(guard) +} + +pub(super) async fn begin_durable_policy_generation( + state: &ServerState, + tenant: &str, + expected_generation: Option, + auth_headers: Option<&HeaderMap>, +) -> Result< + ( + SpecPublicationGuard, + std::sync::Arc, + Vec, + ), + axum::response::Response, +> { + let guard = + begin_policy_generation_mutation(state, tenant, expected_generation, auth_headers).await?; + let store = state.policy_store().ok_or_else(|| { + ( + StatusCode::SERVICE_UNAVAILABLE, + "Persistence backend not configured", + ) + .into_response() + })?; + let rows = load_or_seed_policy_generation(state, store.as_ref(), tenant) + .await + .map_err(|error| { + tracing::warn!(%error, tenant, "failed to load durable policy generation"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to load policies: {error}"), + ) + .into_response() + })?; + let mut entries = rows + .iter() + .map(PolicyGenerationEntry::from) + .collect::>(); + entries.sort_by(|left, right| left.policy_id.cmp(&right.policy_id)); + Ok((guard, store, entries)) +} + +pub(super) fn enabled_named_policies(entries: &[PolicyGenerationEntry]) -> Vec<(String, String)> { + entries + .iter() + .filter(|entry| entry.enabled) + .map(|entry| (entry.policy_id.clone(), entry.cedar_text.clone())) + .collect() +} + +pub(super) fn policy_generation_writes( + entries: &[PolicyGenerationEntry], +) -> Vec { + entries + .iter() + .map(|entry| PolicyGenerationWrite { + policy_id: entry.policy_id.clone(), + cedar_text: entry.cedar_text.clone(), + enabled: entry.enabled, + created_by: entry.created_by.clone(), + }) + .collect() +} + +#[expect( + clippy::result_large_err, + reason = "policy helpers return the fully constructed HTTP rejection response" +)] +pub(super) fn validate_policy_generation( + tenant: &str, + entries: &[PolicyGenerationEntry], +) -> Result<(), axum::response::Response> { + let named = enabled_named_policies(entries); + temper_authz::AuthzEngine::empty() + .reload_tenant_policies_named(tenant, &named) + .map_err(|error| { + tracing::warn!(%error, tenant, "policy validation failed"); + ( + StatusCode::BAD_REQUEST, + format!("Policy validation failed: {error}"), + ) + .into_response() + }) +} + +pub(super) fn policy_generation_intent( + kind: &str, + entries: &[PolicyGenerationEntry], + request_components: &[(&str, &[u8])], +) -> String { + let mut components = request_components + .iter() + .map(|(name, value)| (format!("request:{name}"), value.to_vec())) + .collect::>(); + for entry in entries { + components.push(( + format!("policy:{}:text", entry.policy_id), + entry.cedar_text.as_bytes().to_vec(), + )); + components.push(( + format!("policy:{}:enabled", entry.policy_id), + vec![u8::from(entry.enabled)], + )); + } + ServerState::spec_publication_intent( + kind, + components + .iter() + .map(|(name, value)| (name.as_str(), value.as_slice())), + ) +} + +#[expect( + clippy::result_large_err, + reason = "policy helpers return the fully constructed HTTP rejection response" +)] +pub(super) fn arm_policy_generation( + state: &ServerState, + guard: &mut SpecPublicationGuard, + tenant: &str, + intent: &str, +) -> Result<(), axum::response::Response> { + let tenant_id = TenantId::new(tenant); + state + .arm_spec_publication(guard, &tenant_id, intent) + .map_err(|error| { + tracing::warn!(%error, tenant, "failed to arm policy generation"); + (StatusCode::CONFLICT, error).into_response() + }) +} + +#[expect( + clippy::result_large_err, + reason = "policy helpers return the fully constructed HTTP rejection response" +)] +pub(super) fn activate_policy_generation( + state: &ServerState, + tenant: &str, + entries: &[PolicyGenerationEntry], + guard: &mut SpecPublicationGuard, +) -> Result<(), axum::response::Response> { + let named = enabled_named_policies(entries); + state + .authz + .reload_tenant_policies_named(tenant, &named) + .map_err(|error| { + tracing::error!(%error, tenant, "durable policy generation could not be activated"); + ( + StatusCode::SERVICE_UNAVAILABLE, + format!("Policy generation activation failed: {error}"), + ) + .into_response() + })?; + let combined = state + .authz + .get_tenant_policy_text(tenant) + .unwrap_or_default(); + state + .tenant_policies + .write() + .map_err(|error| { + tracing::error!(%error, tenant, "policy compatibility cache lock poisoned"); + ( + StatusCode::SERVICE_UNAVAILABLE, + "Policy generation activation failed", + ) + .into_response() + })? + .insert(tenant.to_string(), combined); + state + .complete_spec_publication_retry(guard, &TenantId::new(tenant)) + .map_err(|error| { + tracing::error!(%error, tenant, "failed to release completed policy generation"); + (StatusCode::SERVICE_UNAVAILABLE, error).into_response() + })?; + let _ = state + .observe_refresh_tx + .send(crate::state::ObserveRefreshHint::Policies); + Ok(()) +} + +#[expect( + clippy::result_large_err, + reason = "policy helpers return the fully constructed HTTP rejection response" +)] +pub(super) fn publish_memory_policy_generation( + state: &ServerState, + tenant: &str, + policy_text: &str, + kind: &str, + guard: &mut SpecPublicationGuard, +) -> Result<(), axum::response::Response> { + temper_authz::AuthzEngine::new(policy_text).map_err(|error| { + tracing::warn!(%error, tenant, "policy validation failed"); + ( + StatusCode::BAD_REQUEST, + format!("Policy validation failed: {error}"), + ) + .into_response() + })?; + let intent = + ServerState::spec_publication_intent(kind, [("policy-text", policy_text.as_bytes())]); + arm_policy_generation(state, guard, tenant, &intent)?; + super::super::validate_and_reload_policies(state, tenant, policy_text)?; + state + .tenant_policies + .write() + .map_err(|error| { + tracing::error!(%error, tenant, "policy compatibility cache lock poisoned"); + ( + StatusCode::SERVICE_UNAVAILABLE, + "Policy generation activation failed", + ) + .into_response() + })? + .insert(tenant.to_string(), policy_text.to_string()); + state + .complete_spec_publication_retry(guard, &TenantId::new(tenant)) + .map_err(|error| (StatusCode::SERVICE_UNAVAILABLE, error).into_response()) +} + +pub(super) async fn publish_policy_upsert_mode( + state: &ServerState, + tenant: &str, + policy_id: &str, + mutation: PolicyUpsert, + created_by: &str, + expected_generation: Option, + auth_headers: Option<&HeaderMap>, +) -> Result<(), axum::response::Response> { + let (mut guard, _store, mut entries) = + begin_durable_policy_generation(state, tenant, expected_generation, auth_headers).await?; + let existing = entries + .iter() + .position(|entry| entry.policy_id == policy_id); + let existing_text = existing + .map(|index| entries[index].cedar_text.as_str()) + .unwrap_or_default(); + let cedar_text = match mutation { + PolicyUpsert::Replace(text) => text, + PolicyUpsert::AppendRule(rule) if existing_text.is_empty() => rule, + PolicyUpsert::AppendRule(rule) + if existing_text == rule || existing_text.ends_with(&format!("\n{rule}")) => + { + existing_text.to_string() + } + PolicyUpsert::AppendRule(rule) => format!("{existing_text}\n{rule}"), + }; + if let Some(index) = existing { + entries[index].cedar_text.clone_from(&cedar_text); + entries[index].enabled = true; + } else { + entries.push(PolicyGenerationEntry { + policy_id: policy_id.to_string(), + cedar_text: cedar_text.clone(), + enabled: true, + created_by: created_by.to_string(), + }); + } + if let Some(index) = existing { + entries[index].created_by = created_by.to_string(); + } + entries.sort_by(|left, right| left.policy_id.cmp(&right.policy_id)); + validate_policy_generation(tenant, &entries)?; + let intent = policy_generation_intent( + "direct-policy-upsert-v1", + &entries, + &[ + ("policy-id", policy_id.as_bytes()), + ("cedar-text", cedar_text.as_bytes()), + ("created-by", created_by.as_bytes()), + ], + ); + arm_policy_generation(state, &mut guard, tenant, &intent)?; + let writes = policy_generation_writes(&entries); + persist_complete_policy_generation(state, tenant, &writes, policy_id, created_by) + .await + .map_err(|error| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to persist policy: {error}"), + ) + .into_response() + })?; + activate_policy_generation(state, tenant, &entries, &mut guard) +} + +pub(in crate::api) async fn publish_policy_replace_all( + state: &ServerState, + tenant: &str, + cedar_text: &str, + created_by: &str, + expected_generation: Option, + auth_headers: Option<&HeaderMap>, +) -> Result<(), axum::response::Response> { + let (mut guard, _store, _existing) = + begin_durable_policy_generation(state, tenant, expected_generation, auth_headers).await?; + let entries = vec![PolicyGenerationEntry { + policy_id: "primary".to_string(), + cedar_text: cedar_text.to_string(), + enabled: true, + created_by: created_by.to_string(), + }]; + validate_policy_generation(tenant, &entries)?; + let intent = policy_generation_intent( + "direct-policy-replace-all-v1", + &entries, + &[ + ("cedar-text", cedar_text.as_bytes()), + ("created-by", created_by.as_bytes()), + ], + ); + arm_policy_generation(state, &mut guard, tenant, &intent)?; + persist_complete_policy_generation( + state, + tenant, + &policy_generation_writes(&entries), + "primary", + created_by, + ) + .await + .map_err(|error| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to persist policy generation: {error}"), + ) + .into_response() + })?; + activate_policy_generation(state, tenant, &entries, &mut guard) +} + +pub(in crate::api) async fn publish_policy_upsert( + state: &ServerState, + tenant: &str, + policy_id: &str, + cedar_text: &str, + created_by: &str, + expected_generation: Option, + auth_headers: Option<&HeaderMap>, +) -> Result<(), axum::response::Response> { + publish_policy_upsert_mode( + state, + tenant, + policy_id, + PolicyUpsert::Replace(cedar_text.to_string()), + created_by, + expected_generation, + auth_headers, + ) + .await +} diff --git a/crates/temper-server/src/api/policies/tests.rs b/crates/temper-server/src/api/policies/tests.rs new file mode 100644 index 000000000..67238c007 --- /dev/null +++ b/crates/temper-server/src/api/policies/tests.rs @@ -0,0 +1,444 @@ +//! Policy publication regression tests. + +use std::sync::atomic::{AtomicU8, Ordering}; +use std::sync::{Arc, Mutex}; + +use temper_runtime::ActorSystem; +use temper_runtime::tenant::TenantId; +use temper_spec::csdl::parse_csdl; +use temper_store_sim::SimEventStore; + +use super::*; +use crate::storage::{BackendLabel, BoxedEventStore, PolicyStore, PolicyStoreRow, StorageStack}; + +const CSDL_XML: &str = include_str!("../../../../../test-fixtures/specs/model.csdl.xml"); +const OLD_POLICY: &str = "permit(principal, action, resource) when { resource.old == true };"; +const NEW_POLICY: &str = "permit(principal, action, resource) when { resource.new == true };"; +const EXTRA_POLICY: &str = "permit(principal, action, resource) when { resource.extra == true };"; + +#[derive(Default)] +struct FaultPolicyStore { + rows: Mutex>, + compatibility_text: Mutex>, + // 0 = no fault, 1 = fail before write, 2 = fail after write. + next_save_fault: AtomicU8, +} + +impl FaultPolicyStore { + fn with_primary(cedar_text: &str) -> Self { + Self { + rows: Mutex::new(vec![PolicyStoreRow { + tenant: "default".to_string(), + policy_id: "primary".to_string(), + cedar_text: cedar_text.to_string(), + policy_hash: "test-hash".to_string(), + created_at: "1970-01-01T00:00:00Z".to_string(), + created_by: "test".to_string(), + enabled: true, + }]), + compatibility_text: Mutex::new(Some(cedar_text.to_string())), + next_save_fault: AtomicU8::new(0), + } + } + + fn fail_next_save_after_commit(&self) { + self.next_save_fault.store(2, Ordering::SeqCst); + } +} + +#[async_trait::async_trait] +impl PolicyStore for FaultPolicyStore { + async fn replace_policy_generation( + &self, + tenant: &str, + entries: &[crate::storage::PolicyGenerationWrite], + compatibility_text: &str, + ) -> Result<(), String> { + let fault = self.next_save_fault.swap(0, Ordering::SeqCst); + if fault == 1 { + return Err("injected pre-commit policy fault".to_string()); + } + let replacement = entries + .iter() + .map(|entry| PolicyStoreRow { + tenant: tenant.to_string(), + policy_id: entry.policy_id.clone(), + cedar_text: entry.cedar_text.clone(), + policy_hash: "test-hash".to_string(), + created_at: "1970-01-01T00:00:00Z".to_string(), + created_by: entry.created_by.clone(), + enabled: entry.enabled, + }) + .collect::>(); + *self.rows.lock().expect("policy rows lock poisoned") = replacement; + *self + .compatibility_text + .lock() + .expect("policy compatibility lock poisoned") = Some(compatibility_text.to_string()); + if fault == 2 { + return Err("injected ambiguous post-commit policy fault".to_string()); + } + Ok(()) + } + + async fn load_policy_compatibility_text( + &self, + _tenant: &str, + ) -> Result, String> { + Ok(self + .compatibility_text + .lock() + .expect("policy compatibility lock poisoned") + .clone()) + } + + async fn save_policy( + &self, + tenant: &str, + policy_id: &str, + cedar_text: &str, + created_by: &str, + ) -> Result { + let fault = self.next_save_fault.swap(0, Ordering::SeqCst); + if fault == 1 { + return Err("injected pre-commit policy fault".to_string()); + } + let mut rows = self.rows.lock().expect("policy rows lock poisoned"); + if let Some(row) = rows + .iter_mut() + .find(|row| row.tenant == tenant && row.policy_id == policy_id) + { + row.cedar_text = cedar_text.to_string(); + row.created_by = created_by.to_string(); + row.enabled = true; + } else { + rows.push(PolicyStoreRow { + tenant: tenant.to_string(), + policy_id: policy_id.to_string(), + cedar_text: cedar_text.to_string(), + policy_hash: "test-hash".to_string(), + created_at: "1970-01-01T00:00:00Z".to_string(), + created_by: created_by.to_string(), + enabled: true, + }); + } + if fault == 2 { + return Err("injected ambiguous post-commit policy fault".to_string()); + } + Ok(true) + } + + async fn load_policies_for_tenant(&self, tenant: &str) -> Result, String> { + Ok(self + .rows + .lock() + .expect("policy rows lock poisoned") + .iter() + .filter(|row| row.tenant == tenant) + .cloned() + .collect()) + } + + async fn load_all_policies(&self) -> Result, String> { + Ok(self.rows.lock().expect("policy rows lock poisoned").clone()) + } + + async fn toggle_policy_enabled( + &self, + tenant: &str, + policy_id: &str, + enabled: bool, + ) -> Result { + let mut rows = self.rows.lock().expect("policy rows lock poisoned"); + let Some(row) = rows + .iter_mut() + .find(|row| row.tenant == tenant && row.policy_id == policy_id) + else { + return Ok(false); + }; + row.enabled = enabled; + Ok(true) + } + + async fn update_policy_text( + &self, + tenant: &str, + policy_id: &str, + cedar_text: &str, + created_by: &str, + ) -> Result { + let mut rows = self.rows.lock().expect("policy rows lock poisoned"); + let Some(row) = rows + .iter_mut() + .find(|row| row.tenant == tenant && row.policy_id == policy_id) + else { + return Ok(false); + }; + row.cedar_text = cedar_text.to_string(); + row.created_by = created_by.to_string(); + Ok(true) + } + + async fn replace_policy( + &self, + tenant: &str, + policy_id: &str, + cedar_text: &str, + enabled: bool, + created_by: &str, + ) -> Result { + let mut rows = self.rows.lock().expect("policy rows lock poisoned"); + let Some(row) = rows + .iter_mut() + .find(|row| row.tenant == tenant && row.policy_id == policy_id) + else { + return Ok(false); + }; + row.cedar_text = cedar_text.to_string(); + row.enabled = enabled; + row.created_by = created_by.to_string(); + Ok(true) + } + + async fn delete_policy(&self, tenant: &str, policy_id: &str) -> Result<(), String> { + self.rows + .lock() + .expect("policy rows lock poisoned") + .retain(|row| row.tenant != tenant || row.policy_id != policy_id); + Ok(()) + } +} + +fn policy_test_state(store: Arc) -> ServerState { + let csdl = parse_csdl(CSDL_XML).expect("test CSDL should parse"); + let mut state = ServerState::new( + ActorSystem::new("policy-generation-test"), + csdl, + CSDL_XML.to_string(), + ); + let events = Arc::new(SimEventStore::no_faults(238)); + state.set_storage_stack(StorageStack::new( + BackendLabel::Sim, + BoxedEventStore::from_arc(events), + None, + None, + None, + Some(store), + None, + None, + None, + None, + )); + state + .authz + .reload_tenant_policies_named( + "default", + &[("primary".to_string(), OLD_POLICY.to_string())], + ) + .expect("activate initial policy"); + state +} + +#[tokio::test] +async fn ambiguous_policy_commit_stays_gated_until_exact_retry_activates_it() { + let store = Arc::new(FaultPolicyStore::with_primary(OLD_POLICY)); + let state = policy_test_state(Arc::clone(&store)); + let tenant = TenantId::default(); + store.fail_next_save_after_commit(); + + assert!( + publish_policy_upsert(&state, "default", "primary", NEW_POLICY, "test", None, None,) + .await + .is_err() + ); + assert_eq!( + state.authz.get_tenant_policy_text("default").as_deref(), + Some(OLD_POLICY) + ); + assert_eq!( + store + .load_policies_for_tenant("default") + .await + .expect("load committed policy")[0] + .cedar_text, + NEW_POLICY + ); + assert!(state.spec_publication_gated(&tenant)); + + assert!( + publish_policy_upsert( + &state, + "default", + "primary", + "permit(principal, action, resource);", + "test", + None, + None, + ) + .await + .is_err() + ); + assert!(state.spec_publication_gated(&tenant)); + + publish_policy_upsert(&state, "default", "primary", NEW_POLICY, "test", None, None) + .await + .expect("exact retry should finish the durable generation"); + assert_eq!( + state.authz.get_tenant_policy_text("default").as_deref(), + Some(NEW_POLICY) + ); + assert!(!state.spec_publication_gated(&tenant)); +} + +#[tokio::test] +async fn stable_tenant_reader_rejects_policy_generation_without_mutation() { + let store = Arc::new(FaultPolicyStore::with_primary(OLD_POLICY)); + let state = policy_test_state(Arc::clone(&store)); + let tenant = TenantId::default(); + let stable_reader = state.begin_tenant_request(&tenant).await; + + assert!( + publish_policy_upsert(&state, "default", "primary", NEW_POLICY, "test", None, None,) + .await + .is_err() + ); + assert_eq!( + store + .load_policies_for_tenant("default") + .await + .expect("load unchanged policy")[0] + .cedar_text, + OLD_POLICY + ); + assert_eq!( + state.authz.get_tenant_policy_text("default").as_deref(), + Some(OLD_POLICY) + ); + + drop(stable_reader); + publish_policy_upsert(&state, "default", "primary", NEW_POLICY, "test", None, None) + .await + .expect("policy generation should publish after reader exits"); +} + +#[tokio::test] +async fn first_granular_policy_write_seeds_the_complete_legacy_generation() { + let store = Arc::new(FaultPolicyStore::default()); + let state = policy_test_state(Arc::clone(&store)); + let added = "permit(principal, action, resource) when { resource.added == true };"; + + publish_policy_upsert(&state, "default", "added-policy", added, "test", None, None) + .await + .expect("first granular mutation must preserve legacy policy authority"); + + let rows = store + .load_policies_for_tenant("default") + .await + .expect("load migrated generation"); + assert!( + rows.iter() + .any(|row| row.policy_id == "primary" && row.cedar_text.contains("resource.old")), + "legacy grants must be durably represented by the primary migration row" + ); + assert!( + rows.iter() + .any(|row| row.policy_id == "added-policy" && row.cedar_text == added) + ); + let active = state + .authz + .get_tenant_policy_text("default") + .expect("active combined generation"); + assert!(active.contains("resource.old")); + assert!(active.contains("resource.added")); +} + +#[tokio::test] +async fn upserting_a_disabled_policy_reenables_the_same_live_generation() { + let store = Arc::new(FaultPolicyStore::with_primary(OLD_POLICY)); + store + .toggle_policy_enabled("default", "primary", false) + .await + .expect("disable durable primary policy"); + let state = policy_test_state(Arc::clone(&store)); + state + .authz + .reload_tenant_policies_named("default", &[]) + .expect("start with no live policies"); + + publish_policy_upsert(&state, "default", "primary", NEW_POLICY, "test", None, None) + .await + .expect("upsert should durably and live re-enable the policy"); + + let rows = store + .load_policies_for_tenant("default") + .await + .expect("load re-enabled policy"); + assert!(rows[0].enabled, "the durable row must be enabled"); + assert_eq!(rows[0].cedar_text, NEW_POLICY); + assert_eq!( + state.authz.get_tenant_policy_text("default").as_deref(), + Some(NEW_POLICY), + "the generation activated by the successful request must include the row" + ); +} + +#[tokio::test] +async fn durable_replace_all_removes_every_non_primary_policy_row() { + let store = Arc::new(FaultPolicyStore::with_primary(OLD_POLICY)); + store + .save_policy("default", "extra", EXTRA_POLICY, "test") + .await + .expect("seed extra durable policy"); + let state = policy_test_state(Arc::clone(&store)); + + publish_policy_replace_all(&state, "default", NEW_POLICY, "test", None, None) + .await + .expect("replace the durable policy generation"); + + let rows = store + .load_policies_for_tenant("default") + .await + .expect("load replaced generation"); + assert_eq!(rows.len(), 1, "replace-all must remove extra durable rows"); + assert_eq!(rows[0].policy_id, "primary"); + assert_eq!(rows[0].cedar_text, NEW_POLICY); + assert_eq!( + state.authz.get_tenant_policy_text("default").as_deref(), + Some(NEW_POLICY), + "the live generation must match the exact durable replacement" + ); +} + +#[tokio::test] +async fn deleting_the_last_policy_row_cannot_resurrect_the_legacy_generation_on_restart() { + let store = Arc::new(FaultPolicyStore::with_primary(OLD_POLICY)); + let state = policy_test_state(Arc::clone(&store)); + crate::authz::policy_persistence::persist_complete_policy_generation( + &state, + "default", + &[], + "primary", + "test", + ) + .await + .expect("publish explicitly empty generation"); + + let restarted = policy_test_state(Arc::clone(&store)); + crate::authz::load_and_activate_tenant_policies(&restarted, "default").await; + + assert!( + store + .load_policies_for_tenant("default") + .await + .expect("load restarted generation") + .is_empty(), + "an explicitly empty granular generation must remain empty after restart" + ); + assert!( + restarted + .authz + .get_tenant_policy_text("default") + .is_none_or(|text| text.is_empty()), + "stale compatibility text must not be promoted after deleting the last row" + ); +} diff --git a/crates/temper-server/src/authz/mod.rs b/crates/temper-server/src/authz/mod.rs index 118423405..20599eb30 100644 --- a/crates/temper-server/src/authz/mod.rs +++ b/crates/temper-server/src/authz/mod.rs @@ -9,5 +9,7 @@ pub(crate) use helpers::{ DenialInput, GovernedMutationAuth, observe_tenant_scope, record_authz_denial, require_governed_mutation_auth, require_observe_auth, security_context_from_headers, }; -pub use policy_persistence::{load_and_activate_tenant_policies, persist_and_activate_policy}; +pub use policy_persistence::{ + load_and_activate_tenant_policies, persist_and_activate_policy, publish_policy_entry_generation, +}; pub use wasm_gate::{CedarWasmAuthzGate, PermissiveWasmAuthzGate}; diff --git a/crates/temper-server/src/authz/policy_persistence.rs b/crates/temper-server/src/authz/policy_persistence.rs index 2ed6d796b..a14e6ab19 100644 --- a/crates/temper-server/src/authz/policy_persistence.rs +++ b/crates/temper-server/src/authz/policy_persistence.rs @@ -11,9 +11,126 @@ //! Called on tenant registration and at server boot. use temper_runtime::scheduler::sim_now; +use temper_runtime::tenant::TenantId; use tracing::instrument; use crate::state::{ServerState, TrajectoryEntry, TrajectorySource}; +use crate::storage::{PolicyGenerationWrite, PolicyStore, PolicyStoreRow}; + +/// Load the canonical granular generation, migrating a legacy aggregate into +/// its reserved `primary` row before the first granular mutation. +/// +/// A crash after the seed but before the requested mutation is safe: restart +/// sees the same legacy grants in `primary`, and the caller can retry the +/// mutation. No first-granular writer may make the legacy generation vanish. +pub(crate) async fn load_or_seed_policy_generation( + state: &ServerState, + store: &dyn PolicyStore, + tenant: &str, +) -> Result, String> { + let rows = store.load_policies_for_tenant(tenant).await?; + if !rows.is_empty() { + // Granular rows are the canonical generation. The aggregate blob and + // in-memory text are compatibility projections and must never be promoted + // once any owned row exists, even when their deduplicated text differs. + return Ok(rows); + } + + let durable_legacy = store.load_policy_compatibility_text(tenant).await?; + let legacy = durable_legacy.or_else(|| { + state + .tenant_policies + .read() + .ok() + .and_then(|policies| policies.get(tenant).cloned()) + .or_else(|| state.authz.get_tenant_policy_text(tenant)) + }); + let Some(legacy) = legacy.filter(|policy| !policy.trim().is_empty()) else { + return Ok(rows); + }; + + store + .save_policy(tenant, "primary", legacy.trim(), "legacy-migration") + .await?; + let migrated = store.load_policies_for_tenant(tenant).await?; + if !migrated.iter().any(|row| row.policy_id == "primary") { + return Err(format!( + "legacy Cedar generation for tenant '{tenant}' was not durably seeded" + )); + } + Ok(migrated) +} + +/// Persist a complete granular generation and its compatibility projection in +/// one backend transaction, then emit the existing policy-change trajectory. +pub(crate) async fn persist_complete_policy_generation( + state: &ServerState, + tenant: &str, + entries: &[PolicyGenerationWrite], + changed_policy_id: &str, + created_by: &str, +) -> Result<(), String> { + let Some(store) = state.policy_store() else { + return Err("durable policy store not configured".to_string()); + }; + let mut entries = entries.to_vec(); + entries.sort_by(|left, right| left.policy_id.cmp(&right.policy_id)); + let compatibility_text = entries + .iter() + .filter(|entry| entry.enabled) + .map(|entry| entry.cedar_text.as_str()) + .collect::>() + .join("\n"); + store + .replace_policy_generation(tenant, &entries, &compatibility_text) + .await + .map_err(|error| { + tracing::warn!( + %error, + tenant, + policy_id = changed_policy_id, + "failed to persist complete Cedar policy generation" + ); + error + })?; + + let entry = TrajectoryEntry { + timestamp: sim_now().to_rfc3339(), + tenant: tenant.to_string(), + entity_type: "_cedar".to_string(), + entity_id: tenant.to_string(), + action: "policy_saved".to_string(), + success: true, + from_status: None, + to_status: None, + error: None, + agent_id: Some(created_by.to_string()), + session_id: None, + authz_denied: None, + denied_resource: None, + denied_module: None, + source: Some(TrajectorySource::Platform), + spec_governed: Some(false), + agent_type: None, + request_body: None, + intent: None, + matched_policy_ids: None, + }; + if !state.enqueue_trajectory_entry(entry) { + tracing::warn!( + tenant, + policy_id = changed_policy_id, + "failed to enqueue policy_saved trajectory entry" + ); + } + tracing::info!( + tenant, + policy_id = changed_policy_id, + created_by, + "Cedar policy change logged" + ); + Ok(()) +} /// Persist a Cedar policy entry and log a trajectory entry on change. /// @@ -27,9 +144,9 @@ use crate::state::{ServerState, TrajectoryEntry, TrajectorySource}; /// This function only handles durable persistence and observability. /// /// Returns `true` if the policy was written (content changed or new entry), -/// `false` when the hash matched and no write was needed. -/// Returns `false` silently (with a `tracing::debug` log) when no durable store -/// is configured. +/// `false` when the hash matched and no write was needed. Persistence absence +/// and write failures are explicit: policy-generation callers must not publish +/// a live Cedar generation when its durable half did not commit. #[instrument(skip_all, fields(tenant, policy_id, otel.name = "authz.persist_and_activate_policy"))] pub async fn persist_and_activate_policy( state: &ServerState, @@ -37,69 +154,140 @@ pub async fn persist_and_activate_policy( policy_id: &str, cedar_text: &str, created_by: &str, -) -> bool { +) -> Result { let Some(store) = state.policy_store() else { - tracing::debug!( - tenant, - policy_id, - "durable store not configured; skipping policy persistence" - ); - return false; + return Err("durable policy store not configured".to_string()); }; + let rows = load_or_seed_policy_generation(state, store.as_ref(), tenant).await?; + let mut entries = rows + .into_iter() + .map(|row| PolicyGenerationWrite { + policy_id: row.policy_id, + cedar_text: row.cedar_text, + enabled: row.enabled, + created_by: row.created_by, + }) + .collect::>(); + let existing = entries + .iter() + .position(|entry| entry.policy_id == policy_id); + if existing + .is_some_and(|index| entries[index].cedar_text == cedar_text && entries[index].enabled) + { + return Ok(false); + } + if let Some(index) = existing { + entries[index].cedar_text = cedar_text.to_string(); + entries[index].enabled = true; + entries[index].created_by = created_by.to_string(); + } else { + entries.push(PolicyGenerationWrite { + policy_id: policy_id.to_string(), + cedar_text: cedar_text.to_string(), + enabled: true, + created_by: created_by.to_string(), + }); + } + persist_complete_policy_generation(state, tenant, &entries, policy_id, created_by).await?; + Ok(true) +} - let changed = match store - .save_policy(tenant, policy_id, cedar_text, created_by) +/// Publish one generated policy as a complete durable/live tenant generation. +/// +/// This is the non-HTTP owner used by spec-driven governance effects. API +/// handlers perform their authorization inside an equivalent writer before +/// calling their own mutation path; both converge on the same `policy_id`, so +/// retries are idempotent and cannot append duplicate Cedar statements. +pub async fn publish_policy_entry_generation( + state: &ServerState, + tenant: &str, + policy_id: &str, + cedar_text: &str, + created_by: &str, +) -> Result<(), String> { + let tenant_id = TenantId::new(tenant); + let mut guard = state.begin_spec_publication(&tenant_id).await?; + let store = state + .policy_store() + .ok_or_else(|| "durable policy store not configured".to_string())?; + let mut entries = load_or_seed_policy_generation(state, store.as_ref(), tenant) .await + .map_err(|error| format!("failed to load durable policy generation: {error}"))? + .into_iter() + .map(|row| PolicyGenerationWrite { + policy_id: row.policy_id, + cedar_text: row.cedar_text, + enabled: row.enabled, + created_by: row.created_by, + }) + .collect::>(); + if let Some(entry) = entries + .iter_mut() + .find(|entry| entry.policy_id == policy_id) { - Ok(changed) => changed, - Err(e) => { - tracing::warn!( - error = %e, - tenant, - policy_id, - "failed to persist Cedar policy" - ); - return false; - } - }; - - if changed { - // Log a trajectory entry so the policy change is observable in the - // Evolution Engine dashboard and trajectory analytics. - let now = sim_now().to_rfc3339(); - let entry = TrajectoryEntry { - timestamp: now, - tenant: tenant.to_string(), - entity_type: "_cedar".to_string(), - entity_id: tenant.to_string(), - action: "policy_saved".to_string(), - success: true, - from_status: None, - to_status: None, - error: None, - agent_id: Some(created_by.to_string()), - session_id: None, - authz_denied: None, - denied_resource: None, - denied_module: None, - source: Some(TrajectorySource::Platform), - spec_governed: Some(false), - agent_type: None, - request_body: None, - intent: None, - matched_policy_ids: None, - }; - if !state.enqueue_trajectory_entry(entry) { - tracing::warn!( - tenant, - policy_id, - "failed to enqueue policy_saved trajectory entry" - ); + entry.cedar_text = cedar_text.to_string(); + entry.enabled = true; + entry.created_by = created_by.to_string(); + } else { + entries.push(PolicyGenerationWrite { + policy_id: policy_id.to_string(), + cedar_text: cedar_text.to_string(), + enabled: true, + created_by: created_by.to_string(), + }); + } + entries.sort_by(|left, right| left.policy_id.cmp(&right.policy_id)); + let named = entries + .iter() + .filter(|entry| entry.enabled) + .map(|entry| (entry.policy_id.clone(), entry.cedar_text.clone())) + .collect::>(); + let mut validation_text = String::new(); + for (_, text) in &named { + if !validation_text.is_empty() { + validation_text.push('\n'); } - tracing::info!(tenant, policy_id, created_by, "Cedar policy change logged"); + validation_text.push_str(text); } - - changed + temper_authz::AuthzEngine::new(&validation_text) + .map_err(|error| format!("generated policy would invalidate tenant policy set: {error}"))?; + let mut intent_components = vec![ + ("policy-id".to_string(), policy_id.as_bytes().to_vec()), + ("cedar-text".to_string(), cedar_text.as_bytes().to_vec()), + ("created-by".to_string(), created_by.as_bytes().to_vec()), + ]; + intent_components.extend(entries.iter().map(|entry| { + ( + format!("entry:{}", entry.policy_id), + [entry.cedar_text.as_bytes(), &[u8::from(entry.enabled)]].concat(), + ) + })); + let intent = ServerState::spec_publication_intent( + "generated-policy-upsert-v1", + intent_components + .iter() + .map(|(name, value)| (name.as_str(), value.as_slice())), + ); + state.arm_spec_publication(&mut guard, &tenant_id, &intent)?; + persist_complete_policy_generation(state, tenant, &entries, policy_id, created_by).await?; + state + .authz + .reload_tenant_policies_named(tenant, &named) + .map_err(|error| format!("failed to activate generated policy: {error}"))?; + let combined = state + .authz + .get_tenant_policy_text(tenant) + .unwrap_or_default(); + state + .tenant_policies + .write() + .map_err(|error| format!("policy compatibility cache lock poisoned: {error}"))? + .insert(tenant.to_string(), combined); + state.complete_spec_publication_retry(&mut guard, &tenant_id)?; + let _ = state + .observe_refresh_tx + .send(crate::state::ObserveRefreshHint::Policies); + Ok(()) } /// Load all persisted Cedar policies for a tenant and activate them. @@ -116,22 +304,18 @@ pub async fn load_and_activate_tenant_policies(state: &ServerState, tenant: &str return; }; - let rows = match store.load_policies_for_tenant(tenant).await { + let rows = match load_or_seed_policy_generation(state, store.as_ref(), tenant).await { Ok(rows) => rows, Err(e) => { tracing::warn!( error = %e, tenant, - "failed to load Cedar policies from durable `policies` table" + "failed to load or migrate Cedar policies from durable storage" ); return; } }; - if rows.is_empty() { - return; - } - // Build named policy entries for per-policy PolicyId assignment. let enabled_count = rows.iter().filter(|r| r.enabled).count(); let named_policies: Vec<(String, String)> = rows diff --git a/crates/temper-server/src/entity_actor/actor.rs b/crates/temper-server/src/entity_actor/actor.rs index cbf46c9a7..77e79e523 100644 --- a/crates/temper-server/src/entity_actor/actor.rs +++ b/crates/temper-server/src/entity_actor/actor.rs @@ -29,22 +29,114 @@ use temper_jit::table::{Effect, TransitionTable}; use temper_observe::wide_event; use temper_runtime::actor::{Actor, ActorContext, ActorError}; use temper_runtime::persistence::{ - COMPOSITE_EVENT_TYPE, EventMetadata, PersistenceEnvelope, PersistenceError, + COMPOSITE_EVENT_TYPE, CompositeEvent, EventMetadata, IndexReconciliation, JournalBoundary, + PersistenceEnvelope, PersistenceError, SnapshotSourceFence, is_state_materialization_event_for, }; use temper_runtime::scheduler::{sim_now, sim_uuid}; use crate::storage::{BackendLabel, BoxedEventStore}; use super::effects::{ - FieldSyncMode, build_eval_context_with_xref, process_action_with_xref_and_field_mode, - prune_transient_action_fields_from_state, + FieldSyncMode, ProcessResult, ScheduledAction, SpawnRequest, build_eval_context_with_xref, + process_action_with_xref_and_field_mode, prune_transient_action_fields_from_state, }; use super::snapshot_queue::{SnapshotEnqueueOutcome, SnapshotWriteQueue}; use super::types::{ - EntityEvent, EntityMsg, EntityResponse, EntityState, MAX_EVENTS_SINCE_SNAPSHOT, - MAX_ITEMS_PER_ENTITY, + EntityEvent, EntityMsg, EntityResponse, EntityState, MAX_DURABLE_IDEMPOTENCY_KEYS_PER_ENTITY, + MAX_EVENTS_SINCE_SNAPSHOT, MAX_ITEMS_PER_ENTITY, }; +mod field_update_idempotency; +mod field_updates; +mod persistence; +mod recovery; +mod state_materialization; + +#[cfg(test)] +pub(crate) use recovery::recover_entity_state_from_store; +pub(crate) use recovery::{ + CapturedEntitySnapshot, EntityRecoveryContext, StableEntitySource, + recover_entity_state_from_stable_sources, recover_entity_state_with_source_from_store, + stable_entity_source_is_current, +}; + +use field_update_idempotency::field_update_intent_fingerprint; +use field_updates::{ + FIELD_UPDATE_EVENT_TYPE, FIELD_UPDATE_SCHEMA, FIELDS_PATCHED_EVENT_TYPE, + FIELDS_REPLACED_EVENT_TYPE, PersistedFieldUpdate, +}; +#[cfg(test)] +pub(crate) use state_materialization::STATE_MATERIALIZATION_EVENT_TYPE; +use state_materialization::rebase_materialized_idempotency_keys; +pub(crate) use state_materialization::{ + PersistedStateMaterialization, STATE_MATERIALIZATION_SCHEMA, state_materialization_envelope, +}; + +const SNAPSHOT_JOURNAL_SEQUENCE_FIELD: &str = "_temper_snapshot_journal_sequence"; +const POST_DISPATCH_EFFECTS_EVENT_TYPE: &str = "Temper.Internal.PostDispatchEffects.v1"; +const POST_DISPATCH_EFFECTS_SCHEMA: &str = "temper.post-dispatch-effects.v1"; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +struct PersistedPostDispatchEffects { + schema: String, + idempotency_key: String, + custom_effects: Vec, + scheduled_actions: Vec, + spawn_requests: Vec, + source_fields: serde_json::Value, + source_status: String, + source_sequence: u64, +} + +impl PersistedPostDispatchEffects { + fn from_result( + idempotency_key: Option<&str>, + state: &EntityState, + result: &ProcessResult, + ) -> Option { + let idempotency_key = idempotency_key.filter(|key| !key.is_empty())?; + if result.custom_effects.is_empty() + && result.scheduled_actions.is_empty() + && result.spawn_requests.is_empty() + { + return None; + } + Some(Self { + schema: POST_DISPATCH_EFFECTS_SCHEMA.to_string(), + idempotency_key: idempotency_key.to_string(), + custom_effects: result.custom_effects.clone(), + scheduled_actions: result.scheduled_actions.clone(), + spawn_requests: result.spawn_requests.clone(), + source_fields: state.fields.clone(), + source_status: state.status.clone(), + source_sequence: 0, + }) + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum SnapshotProvenance { + Legacy, + LegacyJournal { through_sequence: u64 }, + Journal { through_sequence: u64 }, +} + +struct PersistencePayload<'a> { + event_type: &'a str, + payload: serde_json::Value, + timestamp: chrono::DateTime, + to_status: &'a str, + post_dispatch_effects: Option, +} + +fn durable_conflict_sequence(error: &PersistenceError, snapshot_fallback: u64) -> Option { + match error { + PersistenceError::ConcurrencyViolation { actual, .. } => Some(*actual), + PersistenceError::SnapshotGenerationChanged => Some(snapshot_fallback), + _ => None, + } +} + fn event_budget_workspace_id(state: &EntityState) -> String { if state.entity_type == "Workspace" { return state.entity_id.clone(); @@ -91,6 +183,29 @@ fn duplicate_idempotency_custom_effects( .unwrap_or_default() } +fn fallback_duplicate_idempotency_response( + table: &TransitionTable, + state: &EntityState, + action: &str, + cross_entity_booleans: &BTreeMap, +) -> EntityResponse { + let custom_effects = + duplicate_idempotency_custom_effects(table, state, action, cross_entity_booleans); + let mut response_state = state.clone(); + if !custom_effects.is_empty() { + prune_transient_action_fields_from_state(&mut response_state); + } + EntityResponse { + success: true, + state: response_state, + error: None, + custom_effects, + scheduled_actions: vec![], + spawn_requests: vec![], + spec_governed: true, + } +} + /// The entity actor -- processes actions through a TransitionTable. /// Optionally persists events to the configured backend. Wide events are emitted /// via the OTEL SDK (no-op when OTEL is not initialised). @@ -105,6 +220,13 @@ pub struct EntityActor { initial_fields: serde_json::Value, /// Optional event journal for persistence. None = in-memory only. event_journal: Option, + /// Exact snapshot generation that produced the actor's current durable state. + snapshot_source: Arc>, + /// Exact declared-key activation contract that produced the actor state. + /// + /// This is actor-state provenance, not a live-table lookup: the shared table + /// may hot-swap before an old actor is evicted during a contract cutover. + state_key_contract: Arc>, /// Optional async snapshot writer. Event appends remain synchronous. snapshot_queue: Option>, /// Persistence backend label used for metrics and backend-specific field sync. @@ -115,11 +237,111 @@ pub struct EntityActor { /// executing an action whose `idempotency_key` is set, so dispatch-layer /// retries that race past the caller's timeout cannot double-execute. idempotency_cache: Option>, + /// Canonical PATCH/PUT intent fingerprints retained for memory-only actors. + /// + /// Each fingerprint is bound to the retained token's sequence because token + /// text may be reused after eviction from the shared idempotency window. + in_memory_field_update_intents: Arc>>, /// Object store for field-overflow blob bytes. SQL stores only refs. blob_store: Option, } impl EntityActor { + async fn duplicate_idempotency_response( + &self, + table: &TransitionTable, + state: &EntityState, + action: &str, + cross_entity_booleans: &BTreeMap, + idempotency_key: &str, + ) -> Result { + let Some(sequence) = state + .processed_idempotency_keys + .get(idempotency_key) + .copied() + else { + return Ok(fallback_duplicate_idempotency_response( + table, + state, + action, + cross_entity_booleans, + )); + }; + let Some(store) = self.event_journal.as_ref() else { + return Ok(fallback_duplicate_idempotency_response( + table, + state, + action, + cross_entity_booleans, + )); + }; + let envelopes = store + .read_events_page( + &self.persistence_id(), + sequence.saturating_sub(1), + sequence, + 1, + ) + .await + .map_err(|error| { + ActorError::custom(format!( + "failed to reload durable post-dispatch effects for {}:{} idempotency key '{}': {error}", + self.entity_type, self.entity_id, idempotency_key + )) + })?; + let Some(envelope) = envelopes.first() else { + return Err(ActorError::custom(format!( + "durable idempotency sequence {sequence} is missing for {}:{} key '{}'", + self.entity_type, self.entity_id, idempotency_key + ))); + }; + if envelope.sequence_nr != sequence { + return Err(ActorError::custom(format!( + "durable idempotency sequence mismatch for {}:{} key '{}' (expected {sequence}, found {})", + self.entity_type, self.entity_id, idempotency_key, envelope.sequence_nr + ))); + } + if envelope.event_type != POST_DISPATCH_EFFECTS_EVENT_TYPE { + return Ok(fallback_duplicate_idempotency_response( + table, + state, + action, + cross_entity_booleans, + )); + } + let persisted = serde_json::from_value::( + envelope.payload.clone(), + ) + .map_err(|error| { + ActorError::custom(format!( + "failed to decode durable post-dispatch effects for {}:{} at sequence {sequence}: {error}", + self.entity_type, self.entity_id + )) + })?; + if persisted.schema != POST_DISPATCH_EFFECTS_SCHEMA + || persisted.idempotency_key != idempotency_key + || persisted.source_sequence != sequence + { + return Err(ActorError::custom(format!( + "durable post-dispatch effect identity mismatch for {}:{} key '{}' at sequence {sequence}", + self.entity_type, self.entity_id, idempotency_key + ))); + } + let mut response_state = state.clone(); + response_state.fields = persisted.source_fields; + response_state.status = persisted.source_status; + response_state.sequence_nr = persisted.source_sequence; + Ok(EntityResponse { + success: true, + state: response_state, + error: None, + custom_effects: persisted.custom_effects, + scheduled_actions: persisted.scheduled_actions, + spawn_requests: persisted.spawn_requests, + spec_governed: true, + }) + } + fn build_initial_state( entity_type: &str, entity_id: &str, @@ -166,60 +388,6 @@ impl EntityActor { }) } - /// Serialize actor state for snapshot persistence, excluding recent event history. - /// - /// The stored snapshot is already a segment boundary, so its hot tail budget - /// is reset in the payload. Lifetime sequence/count fields remain intact. - fn serialize_snapshot_state(state: &EntityState) -> Result, PersistenceError> { - let mut value = serde_json::to_value(state) - .map_err(|e| PersistenceError::Serialization(e.to_string()))?; - if let Some(obj) = value.as_object_mut() { - obj.remove("events"); - obj.insert("events_since_snapshot".to_string(), serde_json::json!(0)); - obj.insert( - "last_snapshot_sequence_nr".to_string(), - serde_json::json!(state.sequence_nr), - ); - } - serde_json::to_vec(&value).map_err(|e| PersistenceError::Serialization(e.to_string())) - } - - /// Attempt to load actor state from snapshot payload bytes. - fn apply_snapshot_bytes(state: &mut EntityState, sequence_nr: u64, bytes: &[u8]) -> bool { - let mut value = match serde_json::from_slice::(bytes) { - Ok(v) => v, - Err(_) => return false, - }; - let Some(obj) = value.as_object_mut() else { - return false; - }; - - // Snapshot intentionally excludes in-memory recent history. - obj.insert("events".to_string(), serde_json::json!([])); - if !obj.contains_key("total_event_count") { - obj.insert( - "total_event_count".to_string(), - serde_json::json!(sequence_nr as usize), - ); - } - obj.insert("events_since_snapshot".to_string(), serde_json::json!(0)); - obj.insert( - "last_snapshot_sequence_nr".to_string(), - serde_json::json!(sequence_nr), - ); - - match serde_json::from_value::(value) { - Ok(mut restored) => { - restored.sequence_nr = sequence_nr; - restored.events_since_snapshot = 0; - restored.last_snapshot_sequence_nr = sequence_nr; - *state = restored; - true - } - Err(_) => false, - } - } - /// Create a new entity actor (in-memory only, no persistence). pub fn new( entity_type: impl Into, @@ -234,10 +402,13 @@ impl EntityActor { table, initial_fields, event_journal: None, + snapshot_source: Arc::new(RwLock::new(SnapshotSourceFence::Unchecked)), + state_key_contract: Arc::new(RwLock::new(String::new())), snapshot_queue: None, event_backend: None, trace_id: sim_uuid().to_string(), idempotency_cache: None, + in_memory_field_update_intents: Arc::new(RwLock::new(BTreeMap::new())), blob_store: None, } } @@ -258,10 +429,13 @@ impl EntityActor { table, initial_fields, event_journal: Some(store), + snapshot_source: Arc::new(RwLock::new(SnapshotSourceFence::Unchecked)), + state_key_contract: Arc::new(RwLock::new(String::new())), snapshot_queue: None, event_backend: Some(backend), trace_id: sim_uuid().to_string(), idempotency_cache: None, + in_memory_field_update_intents: Arc::new(RwLock::new(BTreeMap::new())), blob_store: None, } } @@ -296,472 +470,6 @@ impl EntityActor { self.blob_store = blob_store; self } - - async fn persist_overflow_blobs( - blob_store: Option<&crate::blob_store::BlobStore>, - blobs: &[crate::blobs::OverflowBlobWrite], - ) -> Result<(), String> { - let Some(blob_store) = blob_store else { - return Err("field-overflow blobs require a configured object blob store".to_string()); - }; - crate::blobs::put_overflow_blobs(blob_store, blobs).await - } - - /// Persistence ID for this entity: "tenant:EntityType:EntityId". - fn persistence_id(&self) -> String { - format!("{}:{}:{}", self.tenant, self.entity_type, self.entity_id) - } - - fn field_sync_mode_for_backend( - backend: Option, - blob_store: Option<&crate::blob_store::BlobStore>, - ) -> FieldSyncMode { - match backend { - Some(BackendLabel::Turso | BackendLabel::TursoRouted) => { - FieldSyncMode::blob_refs_default() - } - Some(_) if blob_store.is_some() => FieldSyncMode::blob_refs_default(), - _ => FieldSyncMode::InlineTruncate, - } - } - - /// Persist an event to the configured event store. - async fn persist_event( - &self, - store: &BoxedEventStore, - backend: BackendLabel, - persistence_id: &str, - state: &mut EntityState, - event: &EntityEvent, - ) -> Result { - let payload = serde_json::to_value(event) - .map_err(|e| PersistenceError::Serialization(e.to_string()))?; - let envelope = PersistenceEnvelope { - sequence_nr: state.sequence_nr + 1, - event_type: event.action.clone(), - payload, - metadata: EventMetadata { - event_id: sim_uuid(), - causation_id: sim_uuid(), - correlation_id: sim_uuid(), - timestamp: event.timestamp, - actor_id: persistence_id.to_string(), - }, - }; - - // W2 / temper#146: measure append wait — the hypothesis is that - // writer-lock / fsync serialization is a cold-start bottleneck. - // ADR-0153/0155: derive the declared key rows AND the vector-index rows from - // the new state and co-commit them with the journal append, so a keyed read - // is correct without a scan and a kNN read reflects the write deterministically. - let (key_rows, vector_rows, reconcile_vectors) = { - let table = self.table.read().expect("table lock poisoned"); - // The type declares vector paths → the store reconciles this entity's - // vector rows (delete stale + insert current) even when no row is emitted - // this write (a delete transition or a cleared property), so stale rows are - // purged instead of being ranked forever (ADR-0155). - let reconcile_vectors = !table.vectors.is_empty(); - let mut key_rows = Vec::new(); - let mut vector_rows = Vec::new(); - if let Some(field_map) = state.fields.as_object() { - for key in &table.keys { - if let Some(hash) = - crate::key_index::canonical_key_hash(&key.name, &key.properties, field_map) - { - key_rows.push(temper_runtime::persistence::EntityKeyRow { - key_name: key.name.clone(), - key_hash: hash, - }); - } - } - // A soft-deleted (tombstone) entity is never indexed — it emits no - // vector rows, so the reconcile below PURGES any it had, even though - // its embedding field may still be present. Mirrors how the field-index - // projection removes a deleted entity. - let index_vectors = state.status != "Deleted"; - for decl in table.vectors.iter().filter(|_| index_vectors) { - // A vector is indexed only when its property parses to `dims` - // floats AND its model tag is a non-empty string — otherwise the - // path indexes nothing for this entity (like an incomplete key). - let Some(vector) = field_map - .get(&decl.property) - .and_then(|v| crate::vector_index::parse_vector_property(v, decl.dims)) - else { - continue; - }; - let Some(model_tag) = field_map - .get(&decl.model_property) - .and_then(|v| v.as_str()) - .filter(|tag| !tag.is_empty()) - else { - continue; - }; - vector_rows.push(temper_runtime::persistence::EntityVectorRow { - decl_name: decl.name.clone(), - model_tag: model_tag.to_string(), - vector, - }); - } - } - (key_rows, vector_rows, reconcile_vectors) - }; - let append_start = Instant::now(); - let result = store - .append_with_index_rows( - persistence_id, - state.sequence_nr, - &[envelope], - &key_rows, - &vector_rows, - reconcile_vectors, - ) - .await; - crate::runtime_metrics::record_event_store_append_wait( - backend.as_str(), - "append", - append_start.elapsed(), - ); - match result { - Ok(new_seq) => { - state.sequence_nr = new_seq; - tracing::debug!(entity = %state.entity_id, seq = new_seq, "event persisted"); - Ok(new_seq) - } - Err(e) => { - tracing::error!( - entity = %state.entity_id, error = %e, - "failed to persist event — state advanced but not durable" - ); - Err(e) - } - } - } - - /// Save a snapshot when the configured interval is reached. - async fn maybe_save_snapshot( - store: &BoxedEventStore, - snapshot_queue: Option<&Arc>, - persistence_id: &str, - state: &mut EntityState, - ) -> Result<(), PersistenceError> { - if state.sequence_nr == 0 { - return Ok(()); - } - if let Some(queue) = snapshot_queue - && let Some(applied_sequence) = queue.applied_sequence(persistence_id) - && applied_sequence > state.last_snapshot_sequence_nr - { - let applied_sequence = applied_sequence.min(state.sequence_nr); - state.last_snapshot_sequence_nr = applied_sequence; - state.events_since_snapshot = - state.sequence_nr.saturating_sub(applied_sequence) as usize; - } - - let interval = Self::snapshot_interval(); - let pending_sequence = snapshot_queue - .and_then(|queue| queue.pending_sequence(persistence_id)) - .unwrap_or(0); - let latest_snapshot_boundary = state.last_snapshot_sequence_nr.max(pending_sequence); - if state.sequence_nr.saturating_sub(latest_snapshot_boundary) < interval { - return Ok(()); - } - - let snapshot = Self::serialize_snapshot_state(state)?; - if let Some(queue) = snapshot_queue { - match queue.enqueue(persistence_id.to_string(), state.sequence_nr, snapshot) { - SnapshotEnqueueOutcome::Enqueued - | SnapshotEnqueueOutcome::Coalesced - | SnapshotEnqueueOutcome::StaleSkipped => return Ok(()), - SnapshotEnqueueOutcome::Full => { - tracing::warn!( - entity = %state.entity_id, - seq = state.sequence_nr, - "snapshot write queue full; keeping replay tail open" - ); - return Ok(()); - } - } - } - - store - .save_snapshot(persistence_id, state.sequence_nr, &snapshot) - .await?; - state.last_snapshot_sequence_nr = state.sequence_nr; - state.events_since_snapshot = 0; - Ok(()) - } - - /// Replay events from the configured store to rebuild state (called in pre_start). - /// - /// Re-evaluates each event through the `TransitionTable` to reconstruct - /// all state variables (status, counters, booleans). This is option 2 from - /// the replay design: the TransitionTable is the authoritative source of - /// effects, so replay produces the same state as the original execution. - async fn replay_events( - table: &TransitionTable, - store: &BoxedEventStore, - backend: BackendLabel, - state: &mut EntityState, - tenant: &str, - blob_store: Option<&crate::blob_store::BlobStore>, - // When true, a journal read failure PROPAGATES as an error instead of being - // swallowed ("start fresh"). The key-index backfill needs this: it must - // distinguish "entity genuinely has no events" from "could not read the - // journal", or it would watermark a type while a present entity is unkeyed - // (a wrong-absent bug). Actor hydration keeps the lenient default (false). - strict_journal_read: bool, - ) -> Result<(), ActorError> { - let replay_start = Instant::now(); // determinism-ok: wall-clock for production replay duration metric only - let persistence_id = format!("{tenant}:{}:{}", state.entity_type, state.entity_id); - let persistence_id = persistence_id.as_str(); - let mut from_sequence = 0; - let mut loaded_snapshot = false; - - match store.load_snapshot(persistence_id).await { - Ok(Some((snapshot_seq, snapshot_bytes))) => { - if Self::apply_snapshot_bytes(state, snapshot_seq, &snapshot_bytes) { - from_sequence = snapshot_seq; - loaded_snapshot = true; - tracing::info!( - entity = %state.entity_id, - seq = snapshot_seq, - "loaded snapshot before replay" - ); - } else { - tracing::warn!( - entity = %state.entity_id, - seq = snapshot_seq, - "failed to deserialize snapshot, falling back to full replay" - ); - } - } - Ok(None) => {} - Err(e) => { - tracing::warn!( - entity = %state.entity_id, - error = %e, - "failed to load snapshot, falling back to full replay" - ); - } - } - - match store.read_events(persistence_id, from_sequence).await { - Ok(envelopes) => { - if envelopes.len() > MAX_EVENTS_SINCE_SNAPSHOT { - return Err(ActorError::custom(format!( - "snapshot tail replay budget exceeded for {}:{} ({} > {} events since snapshot)", - state.entity_type, - state.entity_id, - envelopes.len(), - MAX_EVENTS_SINCE_SNAPSHOT - ))); - } - for env in &envelopes { - if env.event_type == COMPOSITE_EVENT_TYPE { - state.sequence_nr = env.sequence_nr; - continue; - } - - let parsed_event = serde_json::from_value::(env.payload.clone()); - - // Tombstone is terminal: once deleted, entity must not replay - // into a live state. Stop at the first Deleted event. - if env.event_type == "Deleted" { - let tombstone = parsed_event.unwrap_or_else(|_| EntityEvent { - action: "Deleted".to_string(), - from_status: state.status.clone(), - to_status: "Deleted".to_string(), - timestamp: env.metadata.timestamp, - params: serde_json::json!({}), - idempotency_key: None, - }); - state.status = tombstone.to_status.clone(); - if let Some(obj) = state.fields.as_object_mut() { - obj.insert( - "Status".to_string(), - serde_json::Value::String(state.status.clone()), - ); - } - state.push_event_bounded(tombstone); - state.sequence_nr = env.sequence_nr; - break; - } - - match parsed_event { - Ok(event) => { - // A persisted event is a historical fact: its guard - // already passed at commit time and its `to_status` - // is authoritative. Replay therefore re-derives the - // transition's EFFECTS from the table but never - // re-gates it — guards (especially cross-entity ones, - // whose related-entity context is not reconstructed - // here) must not silently drop committed history. - // `replay_effects` returns the matching rule's - // effects ignoring guards; `None` means the table no - // longer knows this action/from-state, in which case - // the stored `to_status` alone carries the state. - let from_status = event.from_status.clone(); - if let Some(effects) = - table.replay_effects(&state.status, &event.action) - { - let effects = effects.to_vec(); - // Shared effect application — same code as handle() and simulation. - let ( - _custom_effects, - _scheduled_actions, - _spawn_requests, - _schedule_at_requests, - ) = super::effects::apply_effects(state, &effects, &event.params); - } - // Always honor the durably-stored target status. This - // is safe (status is always persisted on the event) - // and is the single source of truth for the post- - // transition state across both the known-action and - // unknown-action cases. - super::effects::apply_new_state_fallback( - state, - &from_status, - &event.to_status, - ); - - // Sync action params into fields — mirrors the live - // process_action() path (effects.rs:155) so data fields - // like Title, Description, Priority survive replay. - let field_sync_mode = - Self::field_sync_mode_for_backend(Some(backend), blob_store); - let overflow_blobs = super::effects::sync_fields_with_metadata( - state, - &event.params, - field_sync_mode, - Some(&table.state_var_metadata), - ); - // Persist replayed overflow blobs so blob-ref envelopes - // resolve on subsequent OData reads. Content-addressed - // dedup makes this idempotent — if the original live - // action already persisted the blob, INSERT OR IGNORE - // is a no-op. If the prior server died between emitting - // the event and persisting the blob, this is the - // recovery path. See ADR-0040, ADR-0045. - if !overflow_blobs.is_empty() - && let Err(e) = - Self::persist_overflow_blobs(blob_store, &overflow_blobs).await - { - tracing::warn!( - entity = %state.entity_id, - error = %e, - overflow_count = overflow_blobs.len(), - "failed to persist replayed overflow blobs — blob-ref envelopes may dangle" - ); - } - - state.push_event_bounded(event); - } - Err(e) => { - // Schema-mismatched event: log and skip rather than panic. - // This preserves entity hydration across spec evolution — - // the last valid state is used and replay continues. - tracing::warn!( - entity = %state.entity_id, - event_id = %env.metadata.event_id, - sequence_nr = env.sequence_nr, - event_type = %env.event_type, - error = %e, - "skipping event with incompatible schema during replay" - ); - tracing::warn!(tenant = %tenant, entity_type = %state.entity_type, "event replay error"); - } - } - state.sequence_nr = env.sequence_nr; - } - if !envelopes.is_empty() { - let replayed_tail = state - .sequence_nr - .saturating_sub(state.last_snapshot_sequence_nr) - as usize; - if replayed_tail > MAX_EVENTS_SINCE_SNAPSHOT { - tracing::error!( - entity = %state.entity_id, - replayed_tail, - cap = MAX_EVENTS_SINCE_SNAPSHOT, - "snapshot tail exceeds bounded replay cap" - ); - } - tracing::info!( - entity = %state.entity_id, - snapshot_loaded = loaded_snapshot, - replayed = envelopes.len(), - status = %state.status, - seq = state.sequence_nr, - total_events = state.total_event_count, - events_since_snapshot = state.events_since_snapshot, - recent_events = state.events.len(), - counters = ?state.counters, - booleans = ?state.booleans, - "state rebuilt from event journal via TransitionTable" - ); - } else if loaded_snapshot { - tracing::info!( - entity = %state.entity_id, - seq = state.sequence_nr, - total_events = state.total_event_count, - events_since_snapshot = state.events_since_snapshot, - "state restored from snapshot (no delta events)" - ); - } - } - Err(e) => { - if strict_journal_read { - return Err(ActorError::custom(format!( - "failed to read events for replay of {}:{}: {e}", - state.entity_type, state.entity_id - ))); - } - tracing::error!( - entity = %state.entity_id, error = %e, - "failed to read events for replay — starting fresh" - ); - } - } - crate::runtime_metrics::record_event_replay_duration( - replay_start.elapsed(), - tenant, - &state.entity_type, - ); - Ok(()) - } -} - -/// Rebuild an entity's current state from its snapshot + event tail. -/// -/// `strict_journal_read`: when true, a journal read failure PROPAGATES as an error -/// instead of being swallowed into a "start fresh"/stale state. The key-index backfill -/// passes `true` so it can tell "no events" apart from "could not read the journal" — -/// keying decisions and the per-type watermark depend on that distinction (ADR-0153 -/// soundness gate). Actor hydration passes `false` (keep serving on a transient read). -#[allow(clippy::too_many_arguments)] -pub(crate) async fn recover_entity_state_from_store( - tenant: &str, - entity_type: &str, - entity_id: &str, - table: &TransitionTable, - store: &BoxedEventStore, - backend: BackendLabel, - initial_fields: &serde_json::Value, - blob_store: Option<&crate::blob_store::BlobStore>, - strict_journal_read: bool, -) -> Result { - let mut state = EntityActor::build_initial_state(entity_type, entity_id, table, initial_fields); - EntityActor::replay_events( - table, - store, - backend, - &mut state, - tenant, - blob_store, - strict_journal_read, - ) - .await?; - Ok(state) } impl Actor for EntityActor { @@ -772,6 +480,7 @@ impl Actor for EntityActor { // Snapshot the table for consistent startup (initial state + replay). // This is a cheap clone — TransitionTable is a few Vecs of strings. let table = self.table.read().expect("table lock poisoned").clone(); + self.record_state_key_contract(&table); let mut state = Self::build_initial_state( &self.entity_type, @@ -784,44 +493,86 @@ impl Actor for EntityActor { // Re-evaluates each event through the TransitionTable to reconstruct // all state variables (status, counters, booleans) — not just item_count. if let (Some(store), Some(backend)) = (self.event_journal.as_ref(), self.event_backend) { - state = recover_entity_state_from_store( - &self.tenant, - &self.entity_type, - &self.entity_id, - &table, - store, - backend, - &self.initial_fields, - self.blob_store.as_ref(), - false, // hydration: keep serving on a transient journal read failure + let recovered = recover_entity_state_with_source_from_store( + EntityRecoveryContext { + tenant: &self.tenant, + entity_type: &self.entity_type, + entity_id: &self.entity_id, + table: &table, + store, + backend, + initial_fields: &self.initial_fields, + blob_store: self.blob_store.as_ref(), + }, + false, // empty captured journals may start fresh; any non-empty or unstable source fails closed ) .await?; + *self + .snapshot_source + .write() + .expect("snapshot source lock poisoned") = recovered.snapshot_source; + state = recovered.state; } // Persist a bootstrap Created event for first-time entities so initial - // fields are durable and replayable. - if self.event_journal.is_some() && state.total_event_count == 0 { - let created = EntityEvent { - action: "Created".to_string(), - from_status: String::new(), - to_status: state.status.clone(), - timestamp: sim_now(), - params: self.initial_fields.clone(), - idempotency_key: None, - }; - - if let (Some(store), Some(backend)) = (self.event_journal.as_ref(), self.event_backend) + // fields are durable and replayable. Snapshot-only migration baselines + // use the same first-journal sequence and exact source fence. + if let (Some(store), Some(backend)) = (self.event_journal.as_ref(), self.event_backend) { + const MAX_BOOTSTRAP_RETRIES: u32 = 2; + let mut retries = 0_u32; + while state.sequence_nr == 0 + && state.total_event_count == 0 + && !matches!( + &*self + .snapshot_source + .read() + .expect("snapshot source lock poisoned"), + SnapshotSourceFence::Exact { .. } + ) { - self.persist_event(store, backend, &self.persistence_id(), &mut state, &created) + let state_before_created = state.clone(); + let created = EntityEvent { + action: "Created".to_string(), + from_status: String::new(), + to_status: state.status.clone(), + timestamp: sim_now(), + params: self.initial_fields.clone(), + idempotency_key: None, + }; + match self + .persist_event( + store, + backend, + &self.persistence_id(), + &table, + &state_before_created, + &mut state, + &created, + None, + ) .await - .map_err(|e| { - ActorError::custom(format!( - "failed to persist bootstrap Created event for {}:{}: {}", - self.entity_type, self.entity_id, e - )) - })?; + { + Ok(_) => { + state.push_event_bounded(created); + break; + } + Err(error) + if durable_conflict_sequence(&error, state.sequence_nr).is_some() + && retries < MAX_BOOTSTRAP_RETRIES => + { + state = self + .recover_authoritative_state(store, backend, &table) + .await?; + retries += 1; + } + Err(error) => { + return Err(ActorError::custom(format!( + "failed to persist bootstrap Created event for {}:{}: {error}", + self.entity_type, self.entity_id + ))); + } + } } - state.push_event_bounded(created); } Ok(state) @@ -870,25 +621,16 @@ impl Actor for EntityActor { if let Some(key) = idempotency_key.as_deref() && state.has_processed_idempotency_key(key) { - let custom_effects = duplicate_idempotency_custom_effects( - &table, - state, - &name, - &cross_entity_booleans, + ctx.reply( + self.duplicate_idempotency_response( + &table, + state, + &name, + &cross_entity_booleans, + key, + ) + .await?, ); - let mut response_state = state.clone(); - if !custom_effects.is_empty() { - prune_transient_action_fields_from_state(&mut response_state); - } - ctx.reply(EntityResponse { - success: true, - state: response_state, - error: None, - custom_effects, - scheduled_actions: vec![], - spawn_requests: vec![], - spec_governed: true, - }); return Ok(()); } @@ -1007,18 +749,34 @@ impl Actor for EntityActor { if let (Some(store), Some(backend)) = (self.event_journal.as_ref(), self.event_backend) { + let post_dispatch_effects = PersistedPostDispatchEffects::from_result( + idempotency_key.as_deref(), + state, + &result, + ); let first_persist = self - .persist_event(store, backend, &self.persistence_id(), state, &event) + .persist_event( + store, + backend, + &self.persistence_id(), + &table, + &state_before, + state, + &event, + post_dispatch_effects, + ) .await; match first_persist { Ok(_) => { // Happy path — fall through to downstream telemetry. } - Err(PersistenceError::ConcurrencyViolation { - expected: _, - actual, - }) => { + Err(error) + if durable_conflict_sequence(&error, state.sequence_nr) + .is_some() => + { + let actual = durable_conflict_sequence(&error, state.sequence_nr) + .expect("guard accepted durable conflict"); // ADR-0046 Sub-Decision 3: dedicated APM span // covering the retry cycle. `attempts` and // `outcome` are recorded at the end so Datadog @@ -1060,22 +818,18 @@ impl Actor for EntityActor { while retry_idx < MAX_RETRIES { retry_idx += 1; - // Rollback speculative state. - *state = state_before.clone(); - - // Catch up to the authoritative sequence. - Self::replay_events( - &table, - store, - backend, - state, - &self.tenant, - self.blob_store.as_ref(), - // Actor hydration keeps the lenient "start - // fresh on read error" behavior (unchanged). - false, - ) - .await?; + // Catch up from a clean initial state. Replaying + // onto the actor's pre-race state would apply + // non-idempotent effects a second time. + *state = self + .recover_authoritative_state(store, backend, &table) + .await + .map_err(|error| { + ActorError::custom(format!( + "failed to catch up {}:{} after action concurrency loss: {error}", + self.entity_type, self.entity_id + )) + })?; // ADR-0046 Sub-Decision 4: replay must at // minimum reach the sequence the store @@ -1088,6 +842,46 @@ impl Actor for EntityActor { (state.sequence_nr={} < last_actual={last_actual})", state.sequence_nr ); + if state.sequence_nr < last_actual { + return Err(ActorError::custom(format!( + "action catch-up under-reached authoritative sequence for {}:{} ({} < {last_actual})", + self.entity_type, self.entity_id, state.sequence_nr + ))); + } + + // A competing writer may have committed this + // exact idempotency key while our stale actor + // was attempting the same action. Durable + // replay is the authority: return the already + // committed result instead of re-evaluating and + // appending the action a second time. + if let Some(key) = idempotency_key.as_deref() + && state.has_processed_idempotency_key(key) + { + let total_attempts = u64::from(1 + retry_idx); + let outcome = crate::runtime_metrics::ConcurrencyRetryOutcome::Success; + retry_span.record("attempts", total_attempts); + retry_span.record("outcome", outcome.as_str()); + crate::runtime_metrics::record_entity_concurrency_retry( + &self.entity_type, + outcome, + total_attempts, + ); + let response = self + .duplicate_idempotency_response( + &table, + state, + &name, + &cross_entity_booleans, + key, + ) + .await?; + if let Some(cache) = self.idempotency_cache.as_ref() { + cache.put(&actor_key, key, response.clone()); + } + ctx.reply(response); + return Ok(()); + } // Refresh baselines so postconditions hold // against the replayed state, not the @@ -1152,13 +946,23 @@ impl Actor for EntityActor { )) .await; // determinism-ok: rare retry backoff (ADR-0046) + let post_dispatch_effects = + PersistedPostDispatchEffects::from_result( + idempotency_key.as_deref(), + state, + &retry_result, + ); + match self .persist_event( store, backend, &self.persistence_id(), + &table, + &state_before, state, &retry_event, + post_dispatch_effects, ) .await { @@ -1173,10 +977,19 @@ impl Actor for EntityActor { )); break; } - Err(PersistenceError::ConcurrencyViolation { - actual: new_actual, - .. - }) if retry_idx < MAX_RETRIES => { + Err(error) + if retry_idx < MAX_RETRIES + && durable_conflict_sequence( + &error, + state.sequence_nr, + ) + .is_some() => + { + let new_actual = durable_conflict_sequence( + &error, + state.sequence_nr, + ) + .expect("guard accepted durable conflict"); // Capture the fresh authoritative // sequence so the next iteration's // post-replay assertion checks @@ -1192,7 +1005,13 @@ impl Actor for EntityActor { ); continue; } - Err(PersistenceError::ConcurrencyViolation { .. }) => { + Err(error) + if durable_conflict_sequence( + &error, + state.sequence_nr, + ) + .is_some() => + { retry_final = Some(( crate::runtime_metrics::ConcurrencyRetryOutcome::Exhausted, Some( @@ -1286,24 +1105,48 @@ impl Actor for EntityActor { wide_event::emit_span(&wide); wide_event::emit_metrics(&wide); + let committed_idempotency_key = event.idempotency_key.clone(); state.push_event_bounded(event); + if self.event_journal.is_some() + && let Some(idempotency_key) = committed_idempotency_key.as_deref() + { + state.record_durable_idempotency_key(idempotency_key, state.sequence_nr); + } + self.record_state_key_contract(&table); let persistence_id = self.persistence_id(); - if let Some(ref store) = self.event_journal - && let Err(e) = Self::maybe_save_snapshot( + if let Some(ref store) = self.event_journal { + let mut snapshot_source = self + .snapshot_source + .read() + .expect("snapshot source lock poisoned") + .clone(); + let key_contract = crate::key_index::declared_key_write_contract(&table); + match Self::maybe_save_snapshot( store, self.snapshot_queue.as_ref(), &persistence_id, state, + &mut snapshot_source, + Some(&key_contract), ) .await - { - tracing::warn!( - entity = %state.entity_id, - seq = state.sequence_nr, - error = %e, - "failed to persist snapshot" - ); + { + Ok(_) => { + *self + .snapshot_source + .write() + .expect("snapshot source lock poisoned") = snapshot_source; + } + Err(e) => { + tracing::warn!( + entity = %state.entity_id, + seq = state.sequence_nr, + error = %e, + "failed to persist snapshot" + ); + } + } } // TigerStyle: Assert postconditions after every transition. @@ -1411,6 +1254,21 @@ impl Actor for EntityActor { spec_governed: true, }); } + EntityMsg::GetPassivationSnapshot => { + ctx.reply(super::types::EntityPassivationSnapshot { + state: state.clone(), + snapshot_source: self + .snapshot_source + .read() + .expect("snapshot source lock poisoned") + .clone(), + key_contract: self + .state_key_contract + .read() + .expect("state key contract lock poisoned") + .clone(), + }); + } EntityMsg::GetField { field } => { let value = state .fields @@ -1419,64 +1277,114 @@ impl Actor for EntityActor { .unwrap_or(serde_json::Value::Null); ctx.reply(value); } - EntityMsg::UpdateFields { fields, replace } => { - if replace { - // PUT: replace all fields (preserve Id and Status) - let id = state.entity_id.clone(); - let status = state.status.clone(); - state.fields = fields; - if let Some(obj) = state.fields.as_object_mut() { - obj.insert("Id".to_string(), serde_json::Value::String(id)); - obj.insert("Status".to_string(), serde_json::Value::String(status)); + EntityMsg::UpdateFields { + fields, + replace, + idempotency_key, + } => { + self.handle_field_update(state, fields, replace, idempotency_key, ctx) + .await?; + } + EntityMsg::Delete => { + const MAX_DELETE_RETRIES: u32 = 2; + let mut retries = 0_u32; + let table = self.table.read().expect("table lock poisoned").clone(); + let deleted = loop { + if state.status == "Deleted" { + ctx.reply(EntityResponse { + success: true, + state: state.clone(), + error: None, + custom_effects: vec![], + scheduled_actions: vec![], + spawn_requests: vec![], + spec_governed: true, + }); + return Ok(()); } - } else { - // PATCH: merge fields into existing - if let (Some(existing), Some(updates)) = - (state.fields.as_object_mut(), fields.as_object()) + if !state.can_accept_event() { + ctx.reply(EntityResponse { + success: false, + state: state.clone(), + error: Some(format!( + "Event budget exhausted ({MAX_EVENTS_SINCE_SNAPSHOT} max since snapshot)" + )), + custom_effects: vec![], + scheduled_actions: vec![], + spawn_requests: vec![], + spec_governed: true, + }); + return Ok(()); + } + let deleted = EntityEvent { + action: "Deleted".to_string(), + from_status: state.status.clone(), + to_status: "Deleted".to_string(), + timestamp: sim_now(), + params: serde_json::json!({}), + idempotency_key: None, + }; + + let (Some(store), Some(backend)) = + (self.event_journal.as_ref(), self.event_backend) + else { + break deleted; + }; + let state_before_delete = state.clone(); + match self + .persist_event( + store, + backend, + &self.persistence_id(), + &table, + &state_before_delete, + state, + &deleted, + None, + ) + .await { - for (k, v) in updates { - existing.insert(k.clone(), v.clone()); + Ok(_) => break deleted, + Err(error) + if durable_conflict_sequence(&error, state.sequence_nr).is_some() + && retries < MAX_DELETE_RETRIES => + { + let actual = durable_conflict_sequence(&error, state.sequence_nr) + .expect("guard accepted durable conflict"); + let recovered = self + .recover_authoritative_state(store, backend, &table) + .await + .map_err(|recovery_error| { + ActorError::custom(format!( + "failed to catch up {}:{} after delete concurrency loss: {recovery_error}", + self.entity_type, self.entity_id + )) + })?; + if recovered.sequence_nr < actual { + return Err(ActorError::custom(format!( + "delete catch-up under-reached authoritative sequence for {}:{} ({} < {actual})", + self.entity_type, self.entity_id, recovered.sequence_nr + ))); + } + *state = recovered; + retries += 1; + continue; + } + Err(error) => { + ctx.reply(EntityResponse { + success: false, + state: state.clone(), + error: Some(format!("persistence failed: {error}")), + custom_effects: vec![], + scheduled_actions: vec![], + spawn_requests: vec![], + spec_governed: true, + }); + return Ok(()); } } - } - ctx.reply(EntityResponse { - success: true, - state: state.clone(), - error: None, - custom_effects: vec![], - scheduled_actions: vec![], - spawn_requests: vec![], - spec_governed: true, - }); - } - EntityMsg::Delete => { - let deleted = EntityEvent { - action: "Deleted".to_string(), - from_status: state.status.clone(), - to_status: "Deleted".to_string(), - timestamp: sim_now(), - params: serde_json::json!({}), - idempotency_key: None, }; - if let (Some(store), Some(backend)) = - (self.event_journal.as_ref(), self.event_backend) - && let Err(e) = self - .persist_event(store, backend, &self.persistence_id(), state, &deleted) - .await - { - ctx.reply(EntityResponse { - success: false, - state: state.clone(), - error: Some(format!("persistence failed: {e}")), - custom_effects: vec![], - scheduled_actions: vec![], - spawn_requests: vec![], - spec_governed: true, - }); - return Ok(()); - } - state.status = deleted.to_status.clone(); if let Some(obj) = state.fields.as_object_mut() { obj.insert( @@ -1485,6 +1393,7 @@ impl Actor for EntityActor { ); } state.push_event_bounded(deleted); + self.record_state_key_contract(&table); ctx.reply(EntityResponse { success: true, diff --git a/crates/temper-server/src/entity_actor/actor/field_update_idempotency.rs b/crates/temper-server/src/entity_actor/actor/field_update_idempotency.rs new file mode 100644 index 000000000..c9400bcbc --- /dev/null +++ b/crates/temper-server/src/entity_actor/actor/field_update_idempotency.rs @@ -0,0 +1,186 @@ +//! Canonical request-intent binding for durable HTTP PATCH/PUT retries. + +use sha2::{Digest, Sha256}; + +use super::field_updates::{ + FIELD_UPDATE_EVENT_TYPE, FIELD_UPDATE_SCHEMA, FIELDS_PATCHED_EVENT_TYPE, + FIELDS_REPLACED_EVENT_TYPE, PersistedFieldUpdate, +}; +use super::*; + +pub(super) fn field_update_intent_fingerprint( + fields: &serde_json::Value, + replace: bool, +) -> Result { + fn canonicalize(value: &serde_json::Value) -> serde_json::Value { + match value { + serde_json::Value::Array(values) => { + serde_json::Value::Array(values.iter().map(canonicalize).collect()) + } + serde_json::Value::Object(object) => { + let mut names = object.keys().collect::>(); + names.sort_unstable(); + let mut canonical = serde_json::Map::new(); + for name in names { + canonical.insert(name.clone(), canonicalize(&object[name])); + } + serde_json::Value::Object(canonical) + } + scalar => scalar.clone(), + } + } + + let canonical_fields = canonicalize(fields); + let canonical_intent = serde_json::to_vec(&(FIELD_UPDATE_SCHEMA, replace, canonical_fields)) + .map_err(|error| { + ActorError::custom(format!( + "failed to serialize field-update intent for idempotency: {error}" + )) + })?; + Ok(format!("{:x}", Sha256::digest(canonical_intent))) +} + +impl EntityActor { + pub(super) fn record_in_memory_field_update_intent( + &self, + state: &EntityState, + idempotency_key: &str, + intent_fingerprint: &str, + ) { + if self.event_journal.is_some() { + return; + } + + let sequence = state + .processed_idempotency_keys + .get(idempotency_key) + .copied() + .expect("field-update event must record its idempotency key"); + let mut intents = self + .in_memory_field_update_intents + .write() + .expect("in-memory field-update intents lock poisoned"); + intents.insert( + idempotency_key.to_string(), + (sequence, intent_fingerprint.to_string()), + ); + while intents.len() > MAX_DURABLE_IDEMPOTENCY_KEYS_PER_ENTITY { + let Some(oldest_key) = intents + .iter() + .min_by_key(|(_, (sequence, _))| *sequence) + .map(|(key, _)| key.clone()) + else { + break; + }; + intents.remove(&oldest_key); + } + } + + pub(super) async fn processed_field_update_matches_intent( + &self, + state: &EntityState, + fields: &serde_json::Value, + replace: bool, + idempotency_key: &str, + intent_fingerprint: &str, + ) -> Result { + let sequence = state + .processed_idempotency_keys + .get(idempotency_key) + .copied() + .ok_or_else(|| { + ActorError::custom(format!( + "processed field-update idempotency key '{}' has no durable sequence for {}:{}", + idempotency_key, self.entity_type, self.entity_id + )) + })?; + + let Some(store) = self.event_journal.as_ref() else { + if let Some((stored_sequence, stored_fingerprint)) = self + .in_memory_field_update_intents + .read() + .expect("in-memory field-update intents lock poisoned") + .get(idempotency_key) + { + return Ok(*stored_sequence == sequence && stored_fingerprint == intent_fingerprint); + } + let Some(event) = state + .events + .iter() + .rev() + .find(|event| event.idempotency_key.as_deref() == Some(idempotency_key)) + else { + return Err(ActorError::custom(format!( + "cannot verify in-memory field-update intent for {}:{} key '{}'", + self.entity_type, self.entity_id, idempotency_key + ))); + }; + let stored_replace = match event.action.as_str() { + FIELDS_PATCHED_EVENT_TYPE => false, + FIELDS_REPLACED_EVENT_TYPE => true, + _ => return Ok(false), + }; + return Ok(stored_replace == replace && event.params == *fields); + }; + + let envelopes = store + .read_events_page( + &self.persistence_id(), + sequence.saturating_sub(1), + sequence, + 1, + ) + .await + .map_err(|error| { + ActorError::custom(format!( + "failed to reload durable field-update intent for {}:{} key '{}': {error}", + self.entity_type, self.entity_id, idempotency_key + )) + })?; + let Some(envelope) = envelopes.first() else { + return Err(ActorError::custom(format!( + "durable field-update idempotency sequence {sequence} is missing for {}:{} key '{}'", + self.entity_type, self.entity_id, idempotency_key + ))); + }; + if envelope.sequence_nr != sequence { + return Err(ActorError::custom(format!( + "durable field-update idempotency sequence mismatch for {}:{} key '{}' (expected {sequence}, found {})", + self.entity_type, self.entity_id, idempotency_key, envelope.sequence_nr + ))); + } + if envelope.event_type != FIELD_UPDATE_EVENT_TYPE { + return Ok(false); + } + + let persisted = + serde_json::from_value::(envelope.payload.clone()).map_err( + |error| { + ActorError::custom(format!( + "failed to decode durable field-update intent for {}:{} at sequence {sequence}: {error}", + self.entity_type, self.entity_id + )) + }, + )?; + if persisted.schema != FIELD_UPDATE_SCHEMA + || persisted.idempotency_key.as_deref() != Some(idempotency_key) + { + return Err(ActorError::custom(format!( + "durable field-update identity mismatch for {}:{} key '{}' at sequence {sequence}", + self.entity_type, self.entity_id, idempotency_key + ))); + } + + let stored_fingerprint = + field_update_intent_fingerprint(&persisted.fields, persisted.replace)?; + if let Some(recorded_fingerprint) = persisted.intent_fingerprint.as_deref() + && recorded_fingerprint != stored_fingerprint + { + return Err(ActorError::custom(format!( + "durable field-update intent fingerprint mismatch for {}:{} key '{}' at sequence {sequence}", + self.entity_type, self.entity_id, idempotency_key + ))); + } + Ok(stored_fingerprint == intent_fingerprint) + } +} diff --git a/crates/temper-server/src/entity_actor/actor/field_updates.rs b/crates/temper-server/src/entity_actor/actor/field_updates.rs new file mode 100644 index 000000000..185b94543 --- /dev/null +++ b/crates/temper-server/src/entity_actor/actor/field_updates.rs @@ -0,0 +1,422 @@ +//! Durable HTTP PATCH/PUT application, persistence, and retry. + +use super::*; + +pub(super) const FIELDS_PATCHED_EVENT_TYPE: &str = "Temper.FieldsPatched"; +pub(super) const FIELDS_REPLACED_EVENT_TYPE: &str = "Temper.FieldsReplaced"; +pub(super) const FIELD_UPDATE_EVENT_TYPE: &str = "Temper.Internal.FieldUpdate.v1"; +pub(super) const FIELD_UPDATE_SCHEMA: &str = "temper.field-update.v1"; + +/// Private journal payload for HTTP PATCH/PUT writes. +/// +/// The event type alone is deliberately insufficient to identify this payload: +/// specs may legally declare an action with the same name. Replay recognizes a +/// field update only when both the internal event type and this schema marker +/// match, otherwise it falls through to normal EntityEvent replay. +#[derive(Debug, serde::Deserialize, serde::Serialize)] +pub(super) struct PersistedFieldUpdate { + pub(super) schema: String, + pub(super) fields: serde_json::Value, + pub(super) replace: bool, + #[serde(default)] + pub(super) idempotency_key: Option, + #[serde(default)] + pub(super) intent_fingerprint: Option, +} + +struct FieldUpdatePersistence<'a> { + fields: &'a serde_json::Value, + replace: bool, + idempotency_key: &'a str, + intent_fingerprint: &'a str, + timestamp: chrono::DateTime, +} + +impl EntityActor { + pub(super) fn apply_field_update( + state: &mut EntityState, + fields: &serde_json::Value, + replace: bool, + ) -> Result<(), String> { + let updates = fields + .as_object() + .ok_or_else(|| "field update payload must be a JSON object".to_string())?; + + if replace { + state.fields = serde_json::Value::Object(updates.clone()); + } else { + let existing = state + .fields + .as_object_mut() + .ok_or_else(|| "entity fields must be a JSON object".to_string())?; + for (name, value) in updates { + existing.insert(name.clone(), value.clone()); + } + } + + let entity_id = state.entity_id.clone(); + let status = state.status.clone(); + let object = state + .fields + .as_object_mut() + .ok_or_else(|| "entity fields must remain a JSON object".to_string())?; + object.insert("Id".to_string(), serde_json::Value::String(entity_id)); + object.insert("Status".to_string(), serde_json::Value::String(status)); + Ok(()) + } + + /// Persist an HTTP PATCH/PUT field update using a private, versioned payload. + async fn persist_field_update( + &self, + store: &BoxedEventStore, + backend: BackendLabel, + table: &TransitionTable, + state_before_update: &EntityState, + state: &mut EntityState, + update: FieldUpdatePersistence<'_>, + ) -> Result { + let payload = serde_json::to_value(PersistedFieldUpdate { + schema: FIELD_UPDATE_SCHEMA.to_string(), + fields: update.fields.clone(), + replace: update.replace, + idempotency_key: Some(update.idempotency_key.to_string()), + intent_fingerprint: Some(update.intent_fingerprint.to_string()), + }) + .map_err(|e| PersistenceError::Serialization(e.to_string()))?; + let to_status = state.status.clone(); + let persistence_id = self.persistence_id(); + self.persist_payload( + store, + backend, + &persistence_id, + table, + state_before_update, + state, + PersistencePayload { + event_type: FIELD_UPDATE_EVENT_TYPE, + payload, + timestamp: update.timestamp, + to_status: &to_status, + post_dispatch_effects: None, + }, + ) + .await + } + + /// Rebuild from a clean initial state after an optimistic-concurrency loss. + /// + /// Replaying onto the actor's speculative/pre-race state would apply the + /// journal twice when no snapshot exists. A fresh rebuild is therefore part + /// of the retry contract, and journal reads are strict: an actor that cannot + /// catch up must stop instead of continuing to serve stale state. + pub(super) async fn recover_authoritative_state( + &self, + store: &BoxedEventStore, + backend: BackendLabel, + table: &TransitionTable, + ) -> Result { + let recovered = recover_entity_state_with_source_from_store( + EntityRecoveryContext { + tenant: &self.tenant, + entity_type: &self.entity_type, + entity_id: &self.entity_id, + table, + store, + backend, + initial_fields: &self.initial_fields, + blob_store: self.blob_store.as_ref(), + }, + true, + ) + .await?; + *self + .snapshot_source + .write() + .expect("snapshot source lock poisoned") = recovered.snapshot_source; + self.record_state_key_contract(table); + Ok(recovered.state) + } + + pub(super) async fn handle_field_update( + &self, + state: &mut EntityState, + fields: serde_json::Value, + replace: bool, + idempotency_key: String, + ctx: &mut ActorContext, + ) -> Result<(), ActorError> { + const MAX_FIELD_UPDATE_RETRIES: u32 = 2; + + let action = if replace { + FIELDS_REPLACED_EVENT_TYPE + } else { + FIELDS_PATCHED_EVENT_TYPE + }; + let timestamp = sim_now(); + let mut retries = 0; + let table = self.table.read().expect("table lock poisoned").clone(); + let intent_fingerprint = field_update_intent_fingerprint(&fields, replace)?; + + loop { + // The append and the actor reply are separate observations. A retry + // may arrive after the private field-update event committed but the + // first ask timed out. Durable replay rebuilds this map from the + // payload below, so the duplicate returns the committed state without + // spending another event-budget slot. + if state.has_processed_idempotency_key(&idempotency_key) { + if !self + .processed_field_update_matches_intent( + state, + &fields, + replace, + &idempotency_key, + &intent_fingerprint, + ) + .await? + { + ctx.reply(EntityResponse { + success: false, + state: state.clone(), + error: Some( + "idempotency key was already used for a different field update intent" + .to_string(), + ), + custom_effects: vec![], + scheduled_actions: vec![], + spawn_requests: vec![], + spec_governed: true, + }); + return Ok(()); + } + ctx.reply(EntityResponse { + success: true, + state: state.clone(), + error: None, + custom_effects: vec![], + scheduled_actions: vec![], + spawn_requests: vec![], + spec_governed: true, + }); + return Ok(()); + } + + // A tombstone is terminal. Persisting a field update after it + // would create a journal suffix replay deliberately never + // applies, leaving the recovered sequence behind the durable + // stream. This check stays inside the retry loop because a + // concurrency catch-up may discover a delete committed by + // another writer. + if state.status == "Deleted" { + ctx.reply(EntityResponse { + success: false, + state: state.clone(), + error: Some("cannot update a deleted entity".to_string()), + custom_effects: vec![], + scheduled_actions: vec![], + spawn_requests: vec![], + spec_governed: true, + }); + return Ok(()); + } + + if !state.can_accept_event() { + let workspace_id = event_budget_workspace_id(state); + crate::event_budget_metrics::record_exhausted( + &self.tenant, + &state.entity_type, + &state.entity_id, + &workspace_id, + ); + tracing::warn!( + tenant = %self.tenant, + entity_type = %state.entity_type, + entity_id = %state.entity_id, + workspace_id = %workspace_id, + status = %state.status, + action, + events_since_snapshot = state.events_since_snapshot, + total_event_count = state.total_event_count, + max_events_since_snapshot = MAX_EVENTS_SINCE_SNAPSHOT, + "Event budget exhausted (10000 max since snapshot)" + ); + ctx.reply(EntityResponse { + success: false, + state: state.clone(), + error: Some(format!( + "Event budget exhausted ({MAX_EVENTS_SINCE_SNAPSHOT} max since snapshot)" + )), + custom_effects: vec![], + scheduled_actions: vec![], + spawn_requests: vec![], + spec_governed: true, + }); + return Ok(()); + } + + let state_before = state.clone(); + if let Err(error) = Self::apply_field_update(state, &fields, replace) { + *state = state_before; + ctx.reply(EntityResponse { + success: false, + state: state.clone(), + error: Some(error), + custom_effects: vec![], + scheduled_actions: vec![], + spawn_requests: vec![], + spec_governed: true, + }); + return Ok(()); + } + + if let (Some(store), Some(backend)) = (self.event_journal.as_ref(), self.event_backend) + { + match self + .persist_field_update( + store, + backend, + &table, + &state_before, + state, + FieldUpdatePersistence { + fields: &fields, + replace, + idempotency_key: &idempotency_key, + intent_fingerprint: &intent_fingerprint, + timestamp, + }, + ) + .await + { + Ok(_) => {} + Err(error) + if durable_conflict_sequence(&error, state.sequence_nr).is_some() => + { + let actual = durable_conflict_sequence(&error, state.sequence_nr) + .expect("guard accepted durable conflict"); + let recovered = self + .recover_authoritative_state(store, backend, &table) + .await + .map_err(|error| { + ActorError::custom(format!( + "failed to catch up {}:{} after field-update concurrency loss: {error}", + self.entity_type, self.entity_id + )) + })?; + if recovered.sequence_nr < actual { + return Err(ActorError::custom(format!( + "field-update catch-up under-reached authoritative sequence for {}:{} ({} < {actual})", + self.entity_type, self.entity_id, recovered.sequence_nr + ))); + } + *state = recovered; + + if retries < MAX_FIELD_UPDATE_RETRIES { + retries += 1; + tracing::warn!( + tenant = %self.tenant, + entity_type = %self.entity_type, + entity_id = %self.entity_id, + action, + actual_seq = actual, + retry = retries, + "field update lost optimistic-concurrency race; retrying from authoritative state" + ); + continue; + } + + ctx.reply(EntityResponse { + success: false, + state: state.clone(), + error: Some( + "persistence failed: optimistic concurrency retry exhausted" + .to_string(), + ), + custom_effects: vec![], + scheduled_actions: vec![], + spawn_requests: vec![], + spec_governed: true, + }); + return Ok(()); + } + Err(error) => { + *state = state_before; + ctx.reply(EntityResponse { + success: false, + state: state.clone(), + error: Some(format!("persistence failed: {error}")), + custom_effects: vec![], + scheduled_actions: vec![], + spawn_requests: vec![], + spec_governed: true, + }); + return Ok(()); + } + } + } + + let event = EntityEvent { + action: action.to_string(), + from_status: state.status.clone(), + to_status: state.status.clone(), + timestamp, + params: fields.clone(), + idempotency_key: Some(idempotency_key.clone()), + }; + state.push_event_bounded(event); + if self.event_journal.is_some() { + state.record_durable_idempotency_key(&idempotency_key, state.sequence_nr); + } else { + self.record_in_memory_field_update_intent( + state, + &idempotency_key, + &intent_fingerprint, + ); + } + self.record_state_key_contract(&table); + let persistence_id = self.persistence_id(); + if let Some(store) = self.event_journal.as_ref() { + let mut snapshot_source = self + .snapshot_source + .read() + .expect("snapshot source lock poisoned") + .clone(); + let key_contract = crate::key_index::declared_key_write_contract(&table); + match Self::maybe_save_snapshot( + store, + self.snapshot_queue.as_ref(), + &persistence_id, + state, + &mut snapshot_source, + Some(&key_contract), + ) + .await + { + Ok(_) => { + *self + .snapshot_source + .write() + .expect("snapshot source lock poisoned") = snapshot_source; + } + Err(error) => { + tracing::warn!( + entity = %state.entity_id, + seq = state.sequence_nr, + error = %error, + "failed to persist snapshot after field update" + ); + } + } + } + ctx.reply(EntityResponse { + success: true, + state: state.clone(), + error: None, + custom_effects: vec![], + scheduled_actions: vec![], + spawn_requests: vec![], + spec_governed: true, + }); + return Ok(()); + } + } +} diff --git a/crates/temper-server/src/entity_actor/actor/persistence.rs b/crates/temper-server/src/entity_actor/actor/persistence.rs new file mode 100644 index 000000000..7bb9e1875 --- /dev/null +++ b/crates/temper-server/src/entity_actor/actor/persistence.rs @@ -0,0 +1,477 @@ +//! Entity journal persistence and snapshot materialization. + +use super::*; + +impl EntityActor { + /// Serialize actor state for snapshot persistence, excluding recent event history. + /// + /// The stored snapshot is already a segment boundary, so its hot tail budget + /// is reset in the payload. Lifetime sequence/count fields remain intact. A + /// journal provenance marker is emitted only when the caller supplies the exact + /// durable journal sequence that produced this state. + pub(crate) fn serialize_snapshot_state( + state: &EntityState, + journal_sequence: Option, + ) -> Result, PersistenceError> { + if let Some(sequence) = journal_sequence + && (sequence == 0 || sequence != state.sequence_nr) + { + return Err(PersistenceError::Serialization(format!( + "snapshot journal provenance {sequence} does not match actor sequence {}", + state.sequence_nr + ))); + } + let mut value = serde_json::to_value(state) + .map_err(|e| PersistenceError::Serialization(e.to_string()))?; + if let Some(obj) = value.as_object_mut() { + obj.remove("events"); + obj.insert("events_since_snapshot".to_string(), serde_json::json!(0)); + obj.insert( + "last_snapshot_sequence_nr".to_string(), + serde_json::json!(state.sequence_nr), + ); + obj.remove(SNAPSHOT_JOURNAL_SEQUENCE_FIELD); + if let Some(sequence) = journal_sequence { + obj.insert( + SNAPSHOT_JOURNAL_SEQUENCE_FIELD.to_string(), + serde_json::json!(sequence), + ); + } + } + serde_json::to_vec(&value).map_err(|e| PersistenceError::Serialization(e.to_string())) + } + + /// Attempt to load actor state from snapshot payload bytes. + pub(super) fn apply_snapshot_bytes( + state: &mut EntityState, + sequence_nr: u64, + bytes: &[u8], + ) -> Option { + let mut value = match serde_json::from_slice::(bytes) { + Ok(v) => v, + Err(_) => return None, + }; + let obj = value.as_object_mut()?; + // Before the explicit provenance field existed, actor-written snapshots + // omitted the bounded in-memory `events` tail and retained the exact + // aggregate sequence. Writers after segmented replay accounting was + // introduced also stored an internally consistent boundary and tail; + // older writers omitted both of those later fields. Generic migration + // snapshots with an `events` field remain provenance-free. + let segmented_coordinates = match ( + obj.get("last_snapshot_sequence_nr") + .and_then(serde_json::Value::as_u64), + obj.get("events_since_snapshot") + .and_then(serde_json::Value::as_u64), + ) { + (Some(last_snapshot_sequence), Some(events_since_snapshot)) => { + last_snapshot_sequence <= sequence_nr + && events_since_snapshot == sequence_nr - last_snapshot_sequence + } + (None, None) => true, + _ => false, + }; + let legacy_journal_sequence = (!obj.contains_key("events") + && obj.get("sequence_nr").and_then(serde_json::Value::as_u64) == Some(sequence_nr) + && segmented_coordinates) + .then_some(sequence_nr); + let provenance = match obj.remove(SNAPSHOT_JOURNAL_SEQUENCE_FIELD) { + None => match legacy_journal_sequence { + Some(through_sequence) => SnapshotProvenance::LegacyJournal { through_sequence }, + None => SnapshotProvenance::Legacy, + }, + Some(value) if value.as_u64() == Some(sequence_nr) => SnapshotProvenance::Journal { + through_sequence: sequence_nr, + }, + Some(_) => return None, + }; + + // Snapshot intentionally excludes in-memory recent history. + obj.insert("events".to_string(), serde_json::json!([])); + if !obj.contains_key("total_event_count") { + obj.insert( + "total_event_count".to_string(), + serde_json::json!(sequence_nr as usize), + ); + } + obj.insert("events_since_snapshot".to_string(), serde_json::json!(0)); + obj.insert( + "last_snapshot_sequence_nr".to_string(), + serde_json::json!(sequence_nr), + ); + + match serde_json::from_value::(value) { + Ok(mut restored) => { + restored.sequence_nr = sequence_nr; + restored.events_since_snapshot = 0; + restored.last_snapshot_sequence_nr = sequence_nr; + *state = restored; + Some(provenance) + } + Err(_) => None, + } + } + + pub(super) async fn persist_overflow_blobs( + blob_store: Option<&crate::blob_store::BlobStore>, + blobs: &[crate::blobs::OverflowBlobWrite], + ) -> Result<(), String> { + let Some(blob_store) = blob_store else { + return Err("field-overflow blobs require a configured object blob store".to_string()); + }; + crate::blobs::put_overflow_blobs(blob_store, blobs).await + } + + /// Persistence ID for this entity: "tenant:EntityType:EntityId". + pub(super) fn persistence_id(&self) -> String { + format!("{}:{}:{}", self.tenant, self.entity_type, self.entity_id) + } + + pub(super) fn record_state_key_contract(&self, table: &TransitionTable) { + *self + .state_key_contract + .write() + .expect("state key contract lock poisoned") = + crate::key_index::declared_key_write_contract(table); + } + + pub(super) fn field_sync_mode_for_backend( + backend: Option, + blob_store: Option<&crate::blob_store::BlobStore>, + ) -> FieldSyncMode { + match backend { + Some(BackendLabel::Turso | BackendLabel::TursoRouted) => { + FieldSyncMode::blob_refs_default() + } + Some(_) if blob_store.is_some() => FieldSyncMode::blob_refs_default(), + _ => FieldSyncMode::InlineTruncate, + } + } + + /// Persist an event to the configured event store. + #[expect( + clippy::too_many_arguments, + reason = "persistence boundary carries both pre-event and candidate state" + )] + pub(super) async fn persist_event( + &self, + store: &BoxedEventStore, + backend: BackendLabel, + persistence_id: &str, + table: &TransitionTable, + state_before_event: &EntityState, + state: &mut EntityState, + event: &EntityEvent, + post_dispatch_effects: Option, + ) -> Result { + let payload = serde_json::to_value(event) + .map_err(|e| PersistenceError::Serialization(e.to_string()))?; + self.persist_payload( + store, + backend, + persistence_id, + table, + state_before_event, + state, + PersistencePayload { + event_type: &event.action, + payload, + timestamp: event.timestamp, + to_status: &event.to_status, + post_dispatch_effects, + }, + ) + .await + } + + #[expect( + clippy::too_many_arguments, + reason = "persistence boundary carries both pre-event and candidate state" + )] + pub(super) async fn persist_payload( + &self, + store: &BoxedEventStore, + backend: BackendLabel, + persistence_id: &str, + table: &TransitionTable, + state_before_event: &EntityState, + state: &mut EntityState, + input: PersistencePayload<'_>, + ) -> Result { + let PersistencePayload { + event_type, + payload, + timestamp, + to_status, + post_dispatch_effects, + } = input; + let envelope = PersistenceEnvelope { + sequence_nr: state.sequence_nr + 1, + event_type: event_type.to_string(), + payload, + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: persistence_id.to_string(), + }, + }; + let snapshot_source = self + .snapshot_source + .read() + .expect("snapshot source lock poisoned") + .clone(); + let materializes_snapshot = + state.sequence_nr == 0 && matches!(&snapshot_source, SnapshotSourceFence::Exact { .. }); + let has_post_dispatch_effects = post_dispatch_effects.is_some(); + let mut envelopes = Vec::with_capacity(3); + if materializes_snapshot { + envelopes.push(state_materialization_envelope( + persistence_id, + state_before_event, + timestamp, + )?); + } + envelopes.push(envelope); + if let Some(mut post_dispatch_effects) = post_dispatch_effects { + let source_sequence = state + .sequence_nr + .checked_add(envelopes.len() as u64) + .and_then(|sequence| sequence.checked_add(1)) + .ok_or_else(|| { + PersistenceError::Storage(format!( + "post-dispatch effect sequence overflow for {persistence_id}" + )) + })?; + post_dispatch_effects.source_sequence = source_sequence; + let payload = serde_json::to_value(post_dispatch_effects) + .map_err(|error| PersistenceError::Serialization(error.to_string()))?; + envelopes.push(PersistenceEnvelope { + sequence_nr: source_sequence, + event_type: POST_DISPATCH_EFFECTS_EVENT_TYPE.to_string(), + payload, + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: persistence_id.to_string(), + }, + }); + } + + // W2 / temper#146: measure append wait — the hypothesis is that + // writer-lock / fsync serialization is a cold-start bottleneck. + // ADR-0153/0155: derive the declared key rows AND the vector-index rows from + // the new state and co-commit them with the journal append, so a keyed read + // is correct without a scan and a kNN read reflects the write deterministically. + let (key_rows, vector_rows, reconciliation) = { + // A keyed type owns exact reconciliation even when its current row set + // is empty (delete, all-null, or otherwise unkeyable). Special Delete + // persists the tombstone before mutating `state.status`, so the event's + // target status is the authoritative deletion signal here. + // Every spec-governed write owns an exact key row set. An empty + // declaration set must still purge rows left by an older contract. + let reconcile_keys = true; + let index_entity = to_status != "Deleted"; + // The type declares vector paths → the store reconciles this entity's + // vector rows (delete stale + insert current) even when no row is emitted + // this write (a delete transition or a cleared property), so stale rows are + // purged instead of being ranked forever (ADR-0155). + let reconcile_vectors = !table.vectors.is_empty(); + let key_rows = + crate::key_index::derive_entity_key_rows(&table.keys, &state.fields, index_entity); + let mut vector_rows = Vec::new(); + if let Some(field_map) = state.fields.as_object() { + // A soft-deleted (tombstone) entity is never indexed — it emits no + // vector rows, so the reconcile below PURGES any it had, even though + // its embedding field may still be present. Mirrors how the field-index + // projection removes a deleted entity. + for decl in table.vectors.iter().filter(|_| index_entity) { + // A vector is indexed only when its property parses to `dims` + // floats AND its model tag is a non-empty string — otherwise the + // path indexes nothing for this entity (like an incomplete key). + let Some(vector) = field_map + .get(&decl.property) + .and_then(|v| crate::vector_index::parse_vector_property(v, decl.dims)) + else { + continue; + }; + let Some(model_tag) = field_map + .get(&decl.model_property) + .and_then(|v| v.as_str()) + .filter(|tag| !tag.is_empty()) + else { + continue; + }; + vector_rows.push(temper_runtime::persistence::EntityVectorRow { + decl_name: decl.name.clone(), + model_tag: model_tag.to_string(), + vector, + }); + } + } + ( + key_rows, + vector_rows, + IndexReconciliation { + keys: reconcile_keys, + key_set_signature: Some(crate::key_index::declared_key_write_contract(table)), + vectors: reconcile_vectors, + snapshot_source, + }, + ) + }; + let append_start = Instant::now(); + let result = store + .append_with_index_rows( + persistence_id, + state.sequence_nr, + &envelopes, + &key_rows, + &vector_rows, + reconciliation, + ) + .await; + crate::runtime_metrics::record_event_store_append_wait( + backend.as_str(), + "append", + append_start.elapsed(), + ); + match result { + Ok(new_seq) => { + state.sequence_nr = new_seq; + if materializes_snapshot { + state.record_internal_envelope(); + rebase_materialized_idempotency_keys(state); + *self + .snapshot_source + .write() + .expect("snapshot source lock poisoned") = SnapshotSourceFence::Absent; + } + if has_post_dispatch_effects { + state.record_internal_envelope(); + } + tracing::debug!(entity = %state.entity_id, seq = new_seq, "event persisted"); + Ok(new_seq) + } + Err(e) => { + tracing::error!( + entity = %state.entity_id, error = %e, + "failed to persist event — state advanced but not durable" + ); + Err(e) + } + } + } + + /// Save a snapshot when the configured interval is reached. + pub(super) async fn maybe_save_snapshot( + store: &BoxedEventStore, + snapshot_queue: Option<&Arc>, + persistence_id: &str, + state: &mut EntityState, + snapshot_source: &mut SnapshotSourceFence, + key_contract: Option<&str>, + ) -> Result>, PersistenceError> { + let applied_sequence = snapshot_queue + .and_then(|queue| queue.applied_sequence_for_contract(persistence_id, key_contract)); + let current_source_sequence = match snapshot_source { + SnapshotSourceFence::Exact { sequence_nr, .. } => Some(*sequence_nr), + SnapshotSourceFence::Absent | SnapshotSourceFence::Unchecked => None, + }; + let applied_source_is_newer = applied_sequence.is_some_and(|sequence_nr| { + current_source_sequence.is_none_or(|current| sequence_nr > current) + }); + if applied_source_is_newer + && let Some(applied_source) = match snapshot_queue { + Some(queue) => { + queue + .applied_source_for_contract(persistence_id, key_contract) + .await + } + None => None, + } + { + let applied_sequence = match &applied_source { + SnapshotSourceFence::Exact { sequence_nr, .. } => *sequence_nr, + SnapshotSourceFence::Absent | SnapshotSourceFence::Unchecked => { + unreachable!("an applied snapshot source is always exact") + } + }; + if current_source_sequence.is_none_or(|sequence_nr| applied_sequence > sequence_nr) { + *snapshot_source = applied_source; + } + } + if state.sequence_nr == 0 { + return Ok(None); + } + if matches!( + snapshot_source, + SnapshotSourceFence::Exact { sequence_nr, .. } if *sequence_nr > state.sequence_nr + ) { + // A migration snapshot may be numerically ahead of the journal. + // Stores correctly treat the lower save as a monotonic no-op, so do + // not enqueue it or claim its attempted bytes as the durable source. + return Ok(None); + } + if let Some(applied_sequence) = applied_sequence + && applied_sequence > state.last_snapshot_sequence_nr + { + let applied_sequence = applied_sequence.min(state.sequence_nr); + state.last_snapshot_sequence_nr = applied_sequence; + state.events_since_snapshot = + state.sequence_nr.saturating_sub(applied_sequence) as usize; + } + + let interval = Self::snapshot_interval(); + let pending_sequence = snapshot_queue + .and_then(|queue| queue.pending_sequence_for_contract(persistence_id, key_contract)) + .unwrap_or(0); + let latest_snapshot_boundary = state.last_snapshot_sequence_nr.max(pending_sequence); + if state.sequence_nr.saturating_sub(latest_snapshot_boundary) < interval { + return Ok(None); + } + + let snapshot = Self::serialize_snapshot_state(state, Some(state.sequence_nr))?; + if let Some(queue) = snapshot_queue { + match queue.enqueue( + persistence_id.to_string(), + state.sequence_nr, + snapshot, + snapshot_source.clone(), + key_contract.map(str::to_string), + ) { + SnapshotEnqueueOutcome::Enqueued + | SnapshotEnqueueOutcome::Coalesced + | SnapshotEnqueueOutcome::StaleSkipped => return Ok(None), + SnapshotEnqueueOutcome::Full => { + tracing::warn!( + entity = %state.entity_id, + seq = state.sequence_nr, + "snapshot write queue full; keeping replay tail open" + ); + return Ok(None); + } + } + } + + store + .save_snapshot_if_source( + persistence_id, + state.sequence_nr, + &snapshot, + snapshot_source, + key_contract, + ) + .await?; + *snapshot_source = SnapshotSourceFence::Exact { + sequence_nr: state.sequence_nr, + state: snapshot.clone(), + }; + state.last_snapshot_sequence_nr = state.sequence_nr; + state.events_since_snapshot = 0; + Ok(Some(snapshot)) + } +} diff --git a/crates/temper-server/src/entity_actor/actor/recovery.rs b/crates/temper-server/src/entity_actor/actor/recovery.rs new file mode 100644 index 000000000..5bf2c9ed4 --- /dev/null +++ b/crates/temper-server/src/entity_actor/actor/recovery.rs @@ -0,0 +1,499 @@ +//! Bounded entity recovery from snapshots and durable journals. + +use super::*; + +mod envelope; +mod stable_source; + +use envelope::apply_replayed_envelope; +pub(crate) use stable_source::{ + CapturedEntitySnapshot, StableEntitySource, recover_entity_state_from_stable_sources, + stable_entity_source_is_current, +}; +use stable_source::{CapturedReplaySource, ReplaySummary}; + +const JOURNAL_REPLAY_PAGE_SIZE: usize = 1_024; +const MAX_STABLE_RECOVERY_ATTEMPTS: usize = 3; + +/// Immutable dependencies and identity shared by entity recovery strategies. +#[derive(Clone, Copy)] +pub(crate) struct EntityRecoveryContext<'a> { + /// Tenant that owns the durable entity. + pub(crate) tenant: &'a str, + /// Durable entity type. + pub(crate) entity_type: &'a str, + /// Durable entity identifier. + pub(crate) entity_id: &'a str, + /// Transition table used to replay domain events. + pub(crate) table: &'a TransitionTable, + /// Event store that owns the durable sources. + pub(crate) store: &'a BoxedEventStore, + /// Backend label used for replay field synchronization and diagnostics. + pub(crate) backend: BackendLabel, + /// Initial fields used before applying durable state. + pub(crate) initial_fields: &'a serde_json::Value, + /// Optional overflow-blob store used while replaying fields. + pub(crate) blob_store: Option<&'a crate::blob_store::BlobStore>, +} + +#[derive(Clone, Copy)] +struct ReplayPolicy { + strict_journal_read: bool, + load_snapshot: bool, + strict_event_decode: bool, + replay_full_journal: bool, +} + +/// Actor state and the exact snapshot generation that participated in recovery. +pub(crate) struct RecoveredEntityState { + /// State reconstructed from one closed durable source generation. + pub(crate) state: EntityState, + /// Exact snapshot source the next derived append must validate atomically. + pub(crate) snapshot_source: SnapshotSourceFence, +} + +fn validate_and_normalize_snapshot_state( + state: &mut EntityState, + entity_type: &str, + entity_id: &str, + table: &TransitionTable, + sequence_nr: u64, +) -> Result<(), ActorError> { + if state.entity_type != entity_type || state.entity_id != entity_id { + return Err(ActorError::custom(format!( + "durable snapshot identity mismatch for {entity_type}:{entity_id} at sequence {sequence_nr}" + ))); + } + // `Deleted` is the runtime's terminal lifecycle marker rather than a + // user-declared IOA state. Snapshot writers may persist that terminal state, + // so validate every other status against the transition table while retaining + // the canonical deletion marker for tombstone/source-precedence recovery. + if state.status != "Deleted" && !table.states.contains(&state.status) { + return Err(ActorError::custom(format!( + "durable snapshot has invalid status '{}' for {entity_type}:{entity_id} at sequence {sequence_nr}", + state.status + ))); + } + let fields = state.fields.as_object_mut().ok_or_else(|| { + ActorError::custom(format!( + "durable snapshot fields are not an object for {entity_type}:{entity_id} at sequence {sequence_nr}" + )) + })?; + fields.insert( + "Id".to_string(), + serde_json::Value::String(entity_id.to_string()), + ); + fields.insert( + "Status".to_string(), + serde_json::Value::String(state.status.clone()), + ); + Ok(()) +} + +async fn replay_events( + context: EntityRecoveryContext<'_>, + state: &mut EntityState, + policy: ReplayPolicy, + captured_source: Option<&CapturedReplaySource>, +) -> Result { + let replay_start = Instant::now(); // determinism-ok: production replay metric only + let persistence_id = format!( + "{}:{}:{}", + context.tenant, state.entity_type, state.entity_id + ); + let mut effective_policy = policy; + let mut from_sequence = 0; + let mut loaded_snapshot = false; + let mut snapshot_provenance = None; + let journal_boundary = match captured_source { + Some(source) => source.journal_boundary, + None => context + .store + .journal_boundary(&persistence_id) + .await + .map_err(|error| { + ActorError::custom(format!( + "failed to read durable journal boundary for {}:{}: {error}", + state.entity_type, state.entity_id + )) + })?, + }; + + if policy.load_snapshot { + let snapshot_result = match captured_source { + Some(source) => Ok(source + .snapshot + .as_ref() + .map(|snapshot| (snapshot.sequence_nr, snapshot.state.clone()))), + None => context.store.load_snapshot(&persistence_id).await, + }; + match snapshot_result { + Ok(Some((snapshot_sequence, snapshot_bytes))) => { + if let Some(provenance) = + EntityActor::apply_snapshot_bytes(state, snapshot_sequence, &snapshot_bytes) + { + validate_and_normalize_snapshot_state( + state, + context.entity_type, + context.entity_id, + context.table, + snapshot_sequence, + )?; + from_sequence = snapshot_sequence; + loaded_snapshot = true; + snapshot_provenance = Some(provenance); + tracing::info!( + entity = %state.entity_id, + seq = snapshot_sequence, + "loaded snapshot before replay" + ); + } else { + if captured_source.is_some() { + return Err(ActorError::custom(format!( + "incompatible durable snapshot schema for {}:{} at sequence {snapshot_sequence}", + state.entity_type, state.entity_id + ))); + } + tracing::warn!( + entity = %state.entity_id, + seq = snapshot_sequence, + "failed to deserialize snapshot, falling back to full replay" + ); + } + } + Ok(None) => {} + Err(error) => { + tracing::warn!( + entity = %state.entity_id, + error = %error, + "failed to load snapshot, falling back to full replay" + ); + } + } + } + + let snapshot_hides_terminal = loaded_snapshot + && state.status != "Deleted" + && journal_boundary + .first_terminal_sequence + .is_some_and(|sequence| sequence <= from_sequence); + let snapshot_lacks_journal_provenance = loaded_snapshot + && journal_boundary.latest_sequence > 0 + && !matches!( + snapshot_provenance, + Some( + SnapshotProvenance::Journal { through_sequence } + | SnapshotProvenance::LegacyJournal { through_sequence } + ) + if through_sequence == from_sequence + && through_sequence <= journal_boundary.latest_sequence + ); + if snapshot_hides_terminal || snapshot_lacks_journal_provenance { + let mut snapshot_fields = state.fields.clone(); + if let Some(fields) = snapshot_fields.as_object_mut() { + fields.insert( + "Status".to_string(), + serde_json::Value::String(context.table.initial_state.clone()), + ); + } + tracing::warn!( + entity = %state.entity_id, + snapshot_sequence = from_sequence, + journal_sequence = journal_boundary.latest_sequence, + snapshot_hides_terminal, + snapshot_lacks_journal_provenance, + "replaying journal from zero over an untrusted snapshot baseline" + ); + *state = EntityActor::build_initial_state( + context.entity_type, + context.entity_id, + context.table, + &snapshot_fields, + ); + from_sequence = 0; + loaded_snapshot = false; + effective_policy.strict_journal_read = true; + effective_policy.strict_event_decode = true; + effective_policy.replay_full_journal = true; + } + + let replay_event_budget = journal_boundary + .latest_sequence + .saturating_sub(from_sequence); + if !effective_policy.replay_full_journal + && replay_event_budget > MAX_EVENTS_SINCE_SNAPSHOT as u64 + { + return Err(ActorError::custom(format!( + "snapshot tail replay budget exceeded for {}:{} ({} > {} events since snapshot)", + state.entity_type, state.entity_id, replay_event_budget, MAX_EVENTS_SINCE_SNAPSHOT + ))); + } + + if replay_event_budget == 0 && effective_policy.strict_journal_read { + let probe = context + .store + .read_events_page( + &persistence_id, + journal_boundary.latest_sequence, + journal_boundary.latest_sequence, + 1, + ) + .await + .map_err(|error| { + ActorError::custom(format!( + "failed to verify journal readability for {}:{}: {error}", + state.entity_type, state.entity_id + )) + })?; + if !probe.is_empty() { + return Err(ActorError::custom(format!( + "journal readability probe for {}:{} crossed durable high-water {}", + state.entity_type, state.entity_id, journal_boundary.latest_sequence + ))); + } + } + + let mut cursor = from_sequence; + let mut replayed_count = 0_u64; + let mut summary = ReplaySummary::default(); + while cursor < journal_boundary.latest_sequence { + let remaining = journal_boundary.latest_sequence - cursor; + let page_len = usize::try_from(remaining.min(JOURNAL_REPLAY_PAGE_SIZE as u64)) + .expect("bounded journal page length fits usize"); + let page = match context + .store + .read_events_page( + &persistence_id, + cursor, + journal_boundary.latest_sequence, + page_len, + ) + .await + { + Ok(page) => page, + Err(error) + if effective_policy.strict_journal_read + || journal_boundary.latest_sequence > from_sequence => + { + return Err(ActorError::custom(format!( + "failed to read events for replay of {}:{}: {error}", + state.entity_type, state.entity_id + ))); + } + Err(error) => { + tracing::error!( + entity = %state.entity_id, + error = %error, + "failed to read events for replay — starting fresh" + ); + break; + } + }; + if page.len() != page_len { + return Err(ActorError::custom(format!( + "journal replay for {}:{} returned {} events where {} were required through durable high-water {}", + state.entity_type, + state.entity_id, + page.len(), + page_len, + journal_boundary.latest_sequence + ))); + } + for (offset, envelope) in page.iter().enumerate() { + let expected_sequence = cursor + offset as u64 + 1; + if envelope.sequence_nr != expected_sequence { + return Err(ActorError::custom(format!( + "journal replay for {}:{} expected sequence {}, received {}", + state.entity_type, state.entity_id, expected_sequence, envelope.sequence_nr + ))); + } + let is_state_materialization = + is_state_materialization_event_for(envelope, &state.entity_type, &state.entity_id); + apply_replayed_envelope( + context.table, + context.backend, + state, + context.tenant, + context.blob_store, + envelope, + effective_policy.strict_event_decode, + ) + .await?; + summary.replayed_state_materialization |= is_state_materialization; + } + cursor = page + .last() + .map(|event| event.sequence_nr) + .expect("validated non-empty journal page"); + replayed_count = replayed_count.saturating_add(page.len() as u64); + } + + if cursor < journal_boundary.latest_sequence { + return Err(ActorError::custom(format!( + "journal replay for {}:{} stopped at sequence {} below durable high-water {}", + state.entity_type, state.entity_id, cursor, journal_boundary.latest_sequence + ))); + } + if journal_boundary.first_terminal_sequence.is_some() + && (state.status != "Deleted" || state.sequence_nr < journal_boundary.latest_sequence) + { + return Err(ActorError::custom(format!( + "journal replay for {}:{} did not preserve terminal history through sequence {}", + state.entity_type, state.entity_id, journal_boundary.latest_sequence + ))); + } + + if replayed_count > 0 { + tracing::info!( + entity = %state.entity_id, + snapshot_loaded = loaded_snapshot, + replayed = replayed_count, + status = %state.status, + seq = state.sequence_nr, + total_events = state.total_event_count, + events_since_snapshot = state.events_since_snapshot, + recent_events = state.events.len(), + counters = ?state.counters, + booleans = ?state.booleans, + "state rebuilt from bounded event-journal pages via TransitionTable" + ); + } else if loaded_snapshot { + tracing::info!( + entity = %state.entity_id, + seq = state.sequence_nr, + total_events = state.total_event_count, + events_since_snapshot = state.events_since_snapshot, + "state restored from snapshot (no delta events)" + ); + } + crate::runtime_metrics::record_event_replay_duration( + replay_start.elapsed(), + context.tenant, + &state.entity_type, + ); + if snapshot_hides_terminal || snapshot_lacks_journal_provenance { + // A legacy generation required one strict migration replay. Do not claim + // a durable snapshot boundary, and retain the raw journal tail that must + // stay within the next restart's bounded replay budget. + state.last_snapshot_sequence_nr = 0; + state.events_since_snapshot = usize::try_from(state.sequence_nr).unwrap_or(usize::MAX); + } + Ok(summary) +} + +async fn capture_replay_source( + context: EntityRecoveryContext<'_>, + persistence_id: &str, +) -> Result { + let journal_boundary = context + .store + .journal_boundary(persistence_id) + .await + .map_err(|error| { + ActorError::custom(format!( + "failed to capture durable journal source for {}:{}: {error}", + context.entity_type, context.entity_id + )) + })?; + let snapshot = context + .store + .load_snapshot(persistence_id) + .await + .map_err(|error| { + ActorError::custom(format!( + "failed to capture durable snapshot source for {}:{}: {error}", + context.entity_type, context.entity_id + )) + })? + .map(|(sequence_nr, state)| CapturedEntitySnapshot { sequence_nr, state }); + Ok(CapturedReplaySource { + journal_boundary, + snapshot, + }) +} + +fn snapshot_source_fence(snapshot: &Option) -> SnapshotSourceFence { + match snapshot { + Some(snapshot) => SnapshotSourceFence::Exact { + sequence_nr: snapshot.sequence_nr, + state: snapshot.state.clone(), + }, + None => SnapshotSourceFence::Absent, + } +} + +/// Rebuild actor state from one exact snapshot/journal generation and close both +/// sources before returning its append fence. +pub(crate) async fn recover_entity_state_with_source_from_store( + context: EntityRecoveryContext<'_>, + strict_journal_read: bool, +) -> Result { + let persistence_id = format!( + "{}:{}:{}", + context.tenant, context.entity_type, context.entity_id + ); + for _attempt in 1..=MAX_STABLE_RECOVERY_ATTEMPTS { + let source = capture_replay_source(context, &persistence_id).await?; + let mut state = EntityActor::build_initial_state( + context.entity_type, + context.entity_id, + context.table, + context.initial_fields, + ); + replay_events( + context, + &mut state, + ReplayPolicy { + strict_journal_read, + load_snapshot: true, + // A stable byte range is not authoritative state if an event in + // that range was silently skipped. Actor recovery fails closed so + // the next append cannot certify derived rows from partial replay. + strict_event_decode: true, + replay_full_journal: false, + }, + Some(&source), + ) + .await?; + let closed = capture_replay_source(context, &persistence_id).await?; + if closed == source { + // Snapshot sequence is an aggregate baseline, while optimistic + // append concurrency is owned by the journal high-water. A + // snapshot-only generation therefore accepts its first event at 1. + state.sequence_nr = source.journal_boundary.latest_sequence; + if state.sequence_nr == 0 && source.snapshot.is_some() { + // The first journal event will materialize this complete baseline. + // Reset replay-budget coordinates to that new journal generation; + // retaining a migration snapshot's aggregate sequence could defer + // snapshots beyond the fixed 10k event budget. + state.last_snapshot_sequence_nr = 0; + state.events_since_snapshot = 0; + } + return Ok(RecoveredEntityState { + state, + snapshot_source: snapshot_source_fence(&source.snapshot), + }); + } + tracing::warn!( + entity = %context.entity_id, + "durable source changed during actor recovery; retrying" + ); + } + Err(ActorError::custom(format!( + "durable source generation for {}:{} did not stabilize after {MAX_STABLE_RECOVERY_ATTEMPTS} attempts", + context.entity_type, context.entity_id + ))) +} + +/// Rebuild an entity's current state from its snapshot plus bounded journal tail. +#[cfg(test)] +pub(crate) async fn recover_entity_state_from_store( + context: EntityRecoveryContext<'_>, + strict_journal_read: bool, +) -> Result { + Ok( + recover_entity_state_with_source_from_store(context, strict_journal_read) + .await? + .state, + ) +} diff --git a/crates/temper-server/src/entity_actor/actor/recovery/envelope.rs b/crates/temper-server/src/entity_actor/actor/recovery/envelope.rs new file mode 100644 index 000000000..36415c5f6 --- /dev/null +++ b/crates/temper-server/src/entity_actor/actor/recovery/envelope.rs @@ -0,0 +1,266 @@ +use super::*; + +pub(super) async fn apply_replayed_envelope( + table: &TransitionTable, + backend: BackendLabel, + state: &mut EntityState, + tenant: &str, + blob_store: Option<&crate::blob_store::BlobStore>, + envelope: &PersistenceEnvelope, + strict_event_decode: bool, +) -> Result<(), ActorError> { + fn advance_durable_tail(state: &mut EntityState, sequence_nr: u64) { + state.sequence_nr = sequence_nr; + state.events_since_snapshot = + usize::try_from(sequence_nr.saturating_sub(state.last_snapshot_sequence_nr)) + .unwrap_or(usize::MAX); + } + + if is_state_materialization_event_for(envelope, &state.entity_type, &state.entity_id) { + let materialization = serde_json::from_value::( + envelope.payload.clone(), + ) + .map_err(|error| { + ActorError::custom(format!( + "failed to decode durable state materialization for {}:{} at sequence {}: {error}", + state.entity_type, state.entity_id, envelope.sequence_nr + )) + })?; + if materialization.schema != STATE_MATERIALIZATION_SCHEMA { + return Err(ActorError::custom(format!( + "unsupported durable state materialization schema '{}' for {}:{} at sequence {}", + materialization.schema, state.entity_type, state.entity_id, envelope.sequence_nr + ))); + } + if envelope.sequence_nr != 1 || state.sequence_nr != 0 { + return Err(ActorError::custom(format!( + "durable state materialization for {}:{} must be the first journal event, found sequence {}", + state.entity_type, state.entity_id, envelope.sequence_nr + ))); + } + if materialization.state.entity_type != state.entity_type + || materialization.state.entity_id != state.entity_id + { + return Err(ActorError::custom(format!( + "durable state materialization identity mismatch for {}:{} at sequence {}", + state.entity_type, state.entity_id, envelope.sequence_nr + ))); + } + if !table.states.contains(&materialization.state.status) { + return Err(ActorError::custom(format!( + "durable state materialization has invalid status '{}' for {}:{} at sequence {}", + materialization.state.status, + state.entity_type, + state.entity_id, + envelope.sequence_nr + ))); + } + if materialization.state.sequence_nr != 0 + || materialization.state.last_snapshot_sequence_nr != 0 + || materialization.state.events_since_snapshot != 0 + || !materialization.state.events.is_empty() + || materialization.state.processed_idempotency_keys.len() + > crate::entity_actor::types::MAX_DURABLE_IDEMPOTENCY_KEYS_PER_ENTITY + { + return Err(ActorError::custom(format!( + "durable state materialization has invalid journal coordinates for {}:{} at sequence {}", + state.entity_type, state.entity_id, envelope.sequence_nr + ))); + } + let materialized_fields = materialization.state.fields.as_object().ok_or_else(|| { + ActorError::custom(format!( + "durable state materialization fields are not an object for {}:{} at sequence {}", + state.entity_type, state.entity_id, envelope.sequence_nr + )) + })?; + if materialized_fields + .get("Id") + .and_then(serde_json::Value::as_str) + != Some(state.entity_id.as_str()) + || materialized_fields + .get("Status") + .and_then(serde_json::Value::as_str) + != Some(materialization.state.status.as_str()) + { + return Err(ActorError::custom(format!( + "durable state materialization field identity mismatch for {}:{} at sequence {}", + state.entity_type, state.entity_id, envelope.sequence_nr + ))); + } + *state = materialization.state; + rebase_materialized_idempotency_keys(state); + advance_durable_tail(state, envelope.sequence_nr); + return Ok(()); + } + + if envelope.event_type == POST_DISPATCH_EFFECTS_EVENT_TYPE { + let persisted = serde_json::from_value::( + envelope.payload.clone(), + ) + .map_err(|error| { + ActorError::custom(format!( + "failed to decode durable post-dispatch effects for {}:{} at sequence {}: {error}", + state.entity_type, state.entity_id, envelope.sequence_nr + )) + })?; + if persisted.schema != POST_DISPATCH_EFFECTS_SCHEMA + || persisted.idempotency_key.is_empty() + || persisted.source_sequence != envelope.sequence_nr + || (persisted.custom_effects.is_empty() + && persisted.scheduled_actions.is_empty() + && persisted.spawn_requests.is_empty()) + { + return Err(ActorError::custom(format!( + "invalid durable post-dispatch effects for {}:{} at sequence {}", + state.entity_type, state.entity_id, envelope.sequence_nr + ))); + } + state.record_durable_idempotency_key(&persisted.idempotency_key, envelope.sequence_nr); + advance_durable_tail(state, envelope.sequence_nr); + return Ok(()); + } + + // Deleted is terminal, but every later legacy/corrupt envelope still consumes + // its durable sequence so the reconstructed state reaches the captured fence. + if state.status == "Deleted" { + advance_durable_tail(state, envelope.sequence_nr); + return Ok(()); + } + // `event_type` is normally the domain action name, so CompositeEvent is not + // a reserved discriminator. Skip only the runtime audit schema; a domain + // action with the same name must continue through ordinary replay, while an + // undecodable payload must still fail strict authoritative recovery. + if envelope.event_type == COMPOSITE_EVENT_TYPE + && let Ok(composite) = serde_json::from_value::(envelope.payload.clone()) + { + advance_durable_tail(state, envelope.sequence_nr); + state.record_durable_idempotency_key( + &composite.composite_idempotency_key, + envelope.sequence_nr, + ); + return Ok(()); + } + + if envelope.event_type == FIELD_UPDATE_EVENT_TYPE + && let Ok(update) = serde_json::from_value::(envelope.payload.clone()) + && update.schema == FIELD_UPDATE_SCHEMA + { + let idempotency_key = update.idempotency_key.clone(); + let status = state.status.clone(); + EntityActor::apply_field_update(state, &update.fields, update.replace).map_err( + |error| { + ActorError::custom(format!( + "failed to replay {} for {}:{}: {error}", + envelope.event_type, state.entity_type, state.entity_id + )) + }, + )?; + state.push_event_bounded(EntityEvent { + action: if update.replace { + FIELDS_REPLACED_EVENT_TYPE.to_string() + } else { + FIELDS_PATCHED_EVENT_TYPE.to_string() + }, + from_status: status.clone(), + to_status: status, + timestamp: envelope.metadata.timestamp, + params: update.fields, + idempotency_key: update.idempotency_key, + }); + advance_durable_tail(state, envelope.sequence_nr); + if let Some(idempotency_key) = idempotency_key.as_deref() { + state.record_durable_idempotency_key(idempotency_key, envelope.sequence_nr); + } + return Ok(()); + } + + let parsed_event = serde_json::from_value::(envelope.payload.clone()); + if envelope.transitions_to_deleted() { + let tombstone = parsed_event.unwrap_or_else(|_| EntityEvent { + action: "Deleted".to_string(), + from_status: state.status.clone(), + to_status: "Deleted".to_string(), + timestamp: envelope.metadata.timestamp, + params: serde_json::json!({}), + idempotency_key: None, + }); + state.status = tombstone.to_status.clone(); + if let Some(fields) = state.fields.as_object_mut() { + fields.insert( + "Status".to_string(), + serde_json::Value::String(state.status.clone()), + ); + } + let idempotency_key = tombstone.idempotency_key.clone(); + state.push_event_bounded(tombstone); + advance_durable_tail(state, envelope.sequence_nr); + if let Some(idempotency_key) = idempotency_key.as_deref() { + state.record_durable_idempotency_key(idempotency_key, envelope.sequence_nr); + } + return Ok(()); + } + + match parsed_event { + Ok(event) => { + // The guard already passed at commit time. Replay the historical + // effects without re-gating, then honor the stored target state. + let from_status = event.from_status.clone(); + if let Some(effects) = table.replay_effects(&state.status, &event.action) { + let effects = effects.to_vec(); + let (_custom_effects, _scheduled_actions, _spawn_requests, _schedule_at_requests) = + crate::entity_actor::effects::apply_effects(state, &effects, &event.params); + } + crate::entity_actor::effects::apply_new_state_fallback( + state, + &from_status, + &event.to_status, + ); + + let field_sync_mode = + EntityActor::field_sync_mode_for_backend(Some(backend), blob_store); + let overflow_blobs = crate::entity_actor::effects::sync_fields_with_metadata( + state, + &event.params, + field_sync_mode, + Some(&table.state_var_metadata), + ); + if !overflow_blobs.is_empty() + && let Err(error) = + EntityActor::persist_overflow_blobs(blob_store, &overflow_blobs).await + { + return Err(ActorError::custom(format!( + "field-overflow blob persistence failed during replay for {}:{} at sequence {}: {error}", + state.entity_type, state.entity_id, envelope.sequence_nr + ))); + } + let idempotency_key = event.idempotency_key.clone(); + state.push_event_bounded(event); + if let Some(idempotency_key) = idempotency_key.as_deref() { + state.record_durable_idempotency_key(idempotency_key, envelope.sequence_nr); + } + } + Err(error) if strict_event_decode => { + return Err(ActorError::custom(format!( + "incompatible durable event schema for {}:{} at sequence {} ({}): {error}", + state.entity_type, state.entity_id, envelope.sequence_nr, envelope.event_type + ))); + } + Err(error) => { + tracing::warn!( + entity = %state.entity_id, + event_id = %envelope.metadata.event_id, + sequence_nr = envelope.sequence_nr, + event_type = %envelope.event_type, + error = %error, + "skipping event with incompatible schema during replay" + ); + tracing::warn!( + tenant = %tenant, + entity_type = %state.entity_type, + "event replay error" + ); + } + } + advance_durable_tail(state, envelope.sequence_nr); + Ok(()) +} diff --git a/crates/temper-server/src/entity_actor/actor/recovery/stable_source.rs b/crates/temper-server/src/entity_actor/actor/recovery/stable_source.rs new file mode 100644 index 000000000..e87a10b83 --- /dev/null +++ b/crates/temper-server/src/entity_actor/actor/recovery/stable_source.rs @@ -0,0 +1,214 @@ +//! Stable snapshot/journal source capture shared by reads and index repair. + +use super::*; + +const MAX_STABLE_SOURCE_ATTEMPTS: usize = 3; + +/// Exact snapshot generation that participated in state reconstruction. +#[derive(Clone, Debug, Eq, PartialEq)] +pub(crate) struct CapturedEntitySnapshot { + /// Snapshot sequence stored alongside its bytes. + pub(crate) sequence_nr: u64, + /// Exact serialized snapshot payload. + pub(crate) state: Vec, +} + +#[derive(Default)] +pub(super) struct ReplaySummary { + pub(super) replayed_state_materialization: bool, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct CapturedReplaySource { + pub(super) journal_boundary: JournalBoundary, + pub(super) snapshot: Option, +} + +/// Entity state reconstructed from one stable durable source generation. +pub(crate) struct StableEntitySource { + /// `None` proves that neither a journal nor a snapshot exists. + pub(crate) state: Option, + /// Exact journal high-water replayed into `state`. + pub(crate) journal_sequence: u64, + /// Snapshot generation used either as the complete snapshot-only state or as + /// the legacy field baseline for a journal-backed state. + pub(crate) snapshot: Option, + /// Whether strict replay applied the internal baseline that materializes a + /// snapshot-only entity into the journal before non-domain audit records. + pub(crate) replayed_state_materialization: bool, +} + +impl StableEntitySource { + /// Authoritative sequence represented by this source generation. + /// + /// A non-empty journal is a distinct generation whose coordinates start at + /// one, so it owns the sequence even when a retired migration snapshot used + /// a numerically larger coordinate. Snapshot sequence is authoritative only + /// while the journal is empty. + pub(crate) fn durable_sequence(&self) -> u64 { + if self.journal_sequence > 0 { + self.journal_sequence + } else { + self.snapshot + .as_ref() + .map(|snapshot| snapshot.sequence_nr) + .unwrap_or(0) + } + } +} + +async fn load_snapshot_strict( + entity_type: &str, + entity_id: &str, + table: &TransitionTable, + store: &BoxedEventStore, + initial_fields: &serde_json::Value, + persistence_id: &str, +) -> Result, ActorError> { + let Some((sequence_nr, bytes)) = + store.load_snapshot(persistence_id).await.map_err(|error| { + ActorError::custom(format!( + "failed to read durable snapshot for {entity_type}:{entity_id}: {error}" + )) + })? + else { + return Ok(None); + }; + let mut state = EntityActor::build_initial_state(entity_type, entity_id, table, initial_fields); + if EntityActor::apply_snapshot_bytes(&mut state, sequence_nr, &bytes).is_none() { + return Err(ActorError::custom(format!( + "incompatible durable snapshot schema for {entity_type}:{entity_id} at sequence {sequence_nr}" + ))); + } + super::validate_and_normalize_snapshot_state( + &mut state, + entity_type, + entity_id, + table, + sequence_nr, + )?; + Ok(Some(( + CapturedEntitySnapshot { + sequence_nr, + state: bytes, + }, + state, + ))) +} + +/// Revalidate a captured source generation without materializing or mutating it. +pub(crate) async fn stable_entity_source_is_current( + store: &BoxedEventStore, + persistence_id: &str, + source: &StableEntitySource, +) -> Result { + let boundary = store + .journal_boundary(persistence_id) + .await + .map_err(|error| { + ActorError::custom(format!( + "failed to close durable journal source {persistence_id}: {error}" + )) + })?; + if boundary.latest_sequence != source.journal_sequence { + return Ok(false); + } + let current_snapshot = store.load_snapshot(persistence_id).await.map_err(|error| { + ActorError::custom(format!( + "failed to close durable snapshot source {persistence_id}: {error}" + )) + })?; + Ok(match (&source.snapshot, current_snapshot) { + (None, None) => true, + (Some(expected), Some((sequence_nr, state))) => { + expected.sequence_nr == sequence_nr && expected.state == state + } + _ => false, + }) +} + +/// Recover one entity from a stable snapshot/journal source generation. +/// +/// A snapshot-only generation is materialized directly. Once a journal exists, +/// a journal-provenance snapshot is the exact replay baseline and only its +/// unsnapshotted tail is applied. Legacy or terminal-hiding snapshots use the +/// same strict full-history migration path as ordinary actor recovery. Both +/// sources are re-read byte-for-byte/high-water-for-high-water before return. +pub(crate) async fn recover_entity_state_from_stable_sources( + context: EntityRecoveryContext<'_>, +) -> Result { + let persistence_id = format!( + "{}:{}:{}", + context.tenant, context.entity_type, context.entity_id + ); + for _attempt in 1..=MAX_STABLE_SOURCE_ATTEMPTS { + let initial_boundary = context + .store + .journal_boundary(&persistence_id) + .await + .map_err(|error| { + ActorError::custom(format!( + "failed to read durable journal source for {}:{}: {error}", + context.entity_type, context.entity_id + )) + })?; + let loaded_snapshot = load_snapshot_strict( + context.entity_type, + context.entity_id, + context.table, + context.store, + context.initial_fields, + &persistence_id, + ) + .await?; + let (snapshot, snapshot_state) = match loaded_snapshot { + Some((snapshot, state)) => (Some(snapshot), Some(state)), + None => (None, None), + }; + + let source = if initial_boundary.latest_sequence == 0 { + StableEntitySource { + state: snapshot_state, + journal_sequence: 0, + snapshot, + replayed_state_materialization: false, + } + } else { + let captured_source = CapturedReplaySource { + journal_boundary: initial_boundary, + snapshot: snapshot.clone(), + }; + let mut state = EntityActor::build_initial_state( + context.entity_type, + context.entity_id, + context.table, + context.initial_fields, + ); + let replay = replay_events( + context, + &mut state, + ReplayPolicy { + strict_journal_read: true, + load_snapshot: true, + strict_event_decode: true, + replay_full_journal: false, + }, + Some(&captured_source), + ) + .await?; + StableEntitySource { + journal_sequence: state.sequence_nr, + state: Some(state), + snapshot, + replayed_state_materialization: replay.replayed_state_materialization, + } + }; + if stable_entity_source_is_current(context.store, &persistence_id, &source).await? { + return Ok(source); + } + } + Err(ActorError::custom(format!( + "durable source generation for {}:{} did not stabilize after {MAX_STABLE_SOURCE_ATTEMPTS} attempts", + context.entity_type, context.entity_id + ))) +} diff --git a/crates/temper-server/src/entity_actor/actor/state_materialization.rs b/crates/temper-server/src/entity_actor/actor/state_materialization.rs new file mode 100644 index 000000000..43719b026 --- /dev/null +++ b/crates/temper-server/src/entity_actor/actor/state_materialization.rs @@ -0,0 +1,157 @@ +//! Durable baseline event for the first journal write after snapshot-only recovery. + +use std::collections::{BTreeSet, VecDeque}; + +use super::*; + +pub(crate) use temper_runtime::persistence::{ + STATE_MATERIALIZATION_EVENT_TYPE, STATE_MATERIALIZATION_PAYLOAD_BYTE_BUDGET, + STATE_MATERIALIZATION_SCHEMA, +}; + +const MATERIALIZED_JOURNAL_SEQUENCE: u64 = 1; + +/// Complete state baseline that makes later journal deltas independently replayable. +#[derive(Debug, serde::Deserialize, serde::Serialize)] +pub(crate) struct PersistedStateMaterialization { + pub(crate) schema: String, + pub(crate) state: EntityState, +} + +#[derive(serde::Serialize)] +struct StateMaterializationRef<'a> { + schema: &'static str, + state: MaterializedEntityStateRef<'a>, +} + +#[derive(serde::Serialize)] +struct MaterializedEntityStateRef<'a> { + entity_type: &'a str, + entity_id: &'a str, + status: &'a str, + item_count: usize, + counters: &'a BTreeMap, + booleans: &'a BTreeMap, + lists: &'a BTreeMap>, + fields: &'a serde_json::Value, + events: &'a [EntityEvent], + total_event_count: usize, + events_since_snapshot: usize, + last_snapshot_sequence_nr: u64, + sequence_nr: u64, + processed_idempotency_keys: &'a BTreeMap, +} + +fn bounded_processed_idempotency_keys(state: &EntityState) -> BTreeMap { + let mut newest = BTreeSet::new(); + for (key, sequence) in &state.processed_idempotency_keys { + newest.insert((*sequence, key.clone())); + if newest.len() > crate::entity_actor::types::MAX_DURABLE_IDEMPOTENCY_KEYS_PER_ENTITY { + newest.pop_first(); + } + } + newest + .into_iter() + .map(|(_, key)| (key, MATERIALIZED_JOURNAL_SEQUENCE)) + .collect() +} + +pub(super) fn rebase_materialized_idempotency_keys(state: &mut EntityState) { + state.processed_idempotency_keys = bounded_processed_idempotency_keys(state); +} + +/// Build the internal event that transfers a snapshot-only generation into the journal. +pub(crate) fn state_materialization_envelope( + persistence_id: &str, + state: &EntityState, + timestamp: chrono::DateTime, +) -> Result { + let processed_idempotency_keys = bounded_processed_idempotency_keys(state); + let materialization = StateMaterializationRef { + schema: STATE_MATERIALIZATION_SCHEMA, + state: MaterializedEntityStateRef { + entity_type: &state.entity_type, + entity_id: &state.entity_id, + status: &state.status, + item_count: state.item_count, + counters: &state.counters, + booleans: &state.booleans, + lists: &state.lists, + fields: &state.fields, + events: &[], + total_event_count: state.total_event_count, + events_since_snapshot: 0, + last_snapshot_sequence_nr: 0, + sequence_nr: 0, + processed_idempotency_keys: &processed_idempotency_keys, + }, + }; + let fits_budget = temper_runtime::persistence::serialized_json_fits_byte_budget( + &materialization, + STATE_MATERIALIZATION_PAYLOAD_BYTE_BUDGET, + ) + .map_err(PersistenceError::Serialization)?; + if !fits_budget { + return Err(PersistenceError::Serialization(format!( + "state materialization byte budget exhausted ({STATE_MATERIALIZATION_PAYLOAD_BYTE_BUDGET} bytes)" + ))); + } + let mut fields = state.fields.clone(); + let fields_object = fields.as_object_mut().ok_or_else(|| { + PersistenceError::Serialization( + "state materialization fields must be a JSON object".to_string(), + ) + })?; + fields_object.insert( + "Id".to_string(), + serde_json::Value::String(state.entity_id.clone()), + ); + fields_object.insert( + "Status".to_string(), + serde_json::Value::String(state.status.clone()), + ); + let materialized_state = EntityState { + entity_type: state.entity_type.clone(), + entity_id: state.entity_id.clone(), + status: state.status.clone(), + item_count: state.item_count, + counters: state.counters.clone(), + booleans: state.booleans.clone(), + lists: state.lists.clone(), + fields, + events: VecDeque::new(), + total_event_count: state.total_event_count, + events_since_snapshot: 0, + last_snapshot_sequence_nr: 0, + sequence_nr: 0, + processed_idempotency_keys, + }; + let persisted = PersistedStateMaterialization { + schema: STATE_MATERIALIZATION_SCHEMA.to_string(), + state: materialized_state, + }; + let fits_budget = temper_runtime::persistence::serialized_json_fits_byte_budget( + &persisted, + STATE_MATERIALIZATION_PAYLOAD_BYTE_BUDGET, + ) + .map_err(PersistenceError::Serialization)?; + if !fits_budget { + return Err(PersistenceError::Serialization(format!( + "state materialization byte budget exhausted ({STATE_MATERIALIZATION_PAYLOAD_BYTE_BUDGET} bytes)" + ))); + } + let payload = serde_json::to_value(persisted) + .map_err(|error| PersistenceError::Serialization(error.to_string()))?; + Ok(PersistenceEnvelope { + sequence_nr: 0, + event_type: STATE_MATERIALIZATION_EVENT_TYPE.to_string(), + payload, + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: persistence_id.to_string(), + }, + }) +} diff --git a/crates/temper-server/src/entity_actor/actor_test.rs b/crates/temper-server/src/entity_actor/actor_test.rs index 0e2f53456..aa242a661 100644 --- a/crates/temper-server/src/entity_actor/actor_test.rs +++ b/crates/temper-server/src/entity_actor/actor_test.rs @@ -40,6 +40,790 @@ module = "scm_ingest_pack" ))) } +#[test] +fn state_materialization_rejects_a_baseline_over_the_serialized_byte_budget() { + let mut state = EntityState { + entity_type: "Document".to_string(), + entity_id: "oversized-baseline".to_string(), + status: "Active".to_string(), + item_count: 0, + counters: BTreeMap::new(), + booleans: BTreeMap::new(), + lists: BTreeMap::new(), + fields: serde_json::json!({ + "Id": "oversized-baseline", + "Status": "Active", + }), + events: std::collections::VecDeque::new(), + total_event_count: 0, + events_since_snapshot: 0, + last_snapshot_sequence_nr: 0, + sequence_nr: 0, + processed_idempotency_keys: BTreeMap::new(), + }; + state.fields["Body"] = serde_json::Value::String("x".repeat(16 * 1024 * 1024)); + + let error = state_materialization_envelope( + "default:Document:oversized-baseline", + &state, + chrono::DateTime::UNIX_EPOCH, + ) + .expect_err("oversized snapshot baselines must fail before cloning the state"); + + assert!( + error + .to_string() + .contains("state materialization byte budget exhausted"), + "unexpected error: {error}" + ); +} + +#[cfg(feature = "sim")] +#[tokio::test] +async fn recovery_fails_closed_when_replayed_overflow_blob_cannot_be_persisted() { + use temper_runtime::persistence::EventStore; + use temper_store_sim::SimEventStore; + + let table = TransitionTable::from_ioa_source( + r#" +[automaton] +name = "Document" +states = ["Active"] +initial = "Active" + +[[action]] +name = "ReplaceBody" +kind = "input" +from = ["Active"] +to = "Active" +params = ["Body"] +"#, + ); + let store = SimEventStore::no_faults(295); + let boxed_store = crate::storage::BoxedEventStore::new(store.clone()); + let persistence_id = "default:Document:overflow-recovery"; + let timestamp = chrono::DateTime::UNIX_EPOCH; + store + .append( + persistence_id, + 0, + &[PersistenceEnvelope { + sequence_nr: 0, + event_type: "ReplaceBody".to_string(), + payload: serde_json::to_value(EntityEvent { + action: "ReplaceBody".to_string(), + from_status: "Active".to_string(), + to_status: "Active".to_string(), + timestamp, + params: serde_json::json!({"Body": "x".repeat(200 * 1024)}), + idempotency_key: None, + }) + .expect("encode replay event"), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: persistence_id.to_string(), + }, + }], + ) + .await + .expect("seed durable event"); + + let result = recover_entity_state_from_store( + EntityRecoveryContext { + tenant: "default", + entity_type: "Document", + entity_id: "overflow-recovery", + table: &table, + store: &boxed_store, + backend: BackendLabel::Turso, + initial_fields: &serde_json::json!({}), + blob_store: None, + }, + true, + ) + .await; + + let error = result.expect_err("recovery must not publish dangling blob references"); + assert!( + error + .to_string() + .contains("field-overflow blob persistence failed during replay"), + "unexpected recovery error: {error}" + ); +} + +#[cfg(feature = "sim")] +#[tokio::test] +async fn materialized_unsnapshotted_durable_tail_restarts_at_the_admitted_budget() { + use temper_runtime::persistence::EventStore; + use temper_store_sim::SimEventStore; + + let table = TransitionTable::from_ioa_source( + r#" +[automaton] +name = "Meter" +states = ["Active"] +initial = "Active" + +[[action]] +name = "Touch" +kind = "input" +from = ["Active"] +to = "Active" +"#, + ); + let store = SimEventStore::no_faults(294); + let boxed_store = crate::storage::BoxedEventStore::new(store.clone()); + let persistence_id = "default:Meter:materialized-budget"; + let timestamp = chrono::DateTime::UNIX_EPOCH; + let baseline = EntityState { + entity_type: "Meter".to_string(), + entity_id: "materialized-budget".to_string(), + status: "Active".to_string(), + item_count: 0, + counters: BTreeMap::new(), + booleans: BTreeMap::new(), + lists: BTreeMap::new(), + fields: serde_json::json!({"Id": "materialized-budget", "Status": "Active"}), + events: std::collections::VecDeque::new(), + total_event_count: 0, + events_since_snapshot: 0, + last_snapshot_sequence_nr: 0, + sequence_nr: 0, + processed_idempotency_keys: BTreeMap::new(), + }; + let mut envelopes = Vec::with_capacity(MAX_EVENTS_SINCE_SNAPSHOT); + envelopes.push( + state_materialization_envelope(persistence_id, &baseline, timestamp) + .expect("encode state materialization"), + ); + for _ in 1..MAX_EVENTS_SINCE_SNAPSHOT { + envelopes.push(PersistenceEnvelope { + sequence_nr: 0, + event_type: "Touch".to_string(), + payload: serde_json::to_value(EntityEvent { + action: "Touch".to_string(), + from_status: "Active".to_string(), + to_status: "Active".to_string(), + timestamp, + params: serde_json::json!({}), + idempotency_key: None, + }) + .expect("encode Touch event"), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: persistence_id.to_string(), + }, + }); + } + store + .append(persistence_id, 0, &envelopes) + .await + .expect("seed the largest admitted unsnapshotted durable tail"); + + let recovered = recover_entity_state_with_source_from_store( + EntityRecoveryContext { + tenant: "default", + entity_type: "Meter", + entity_id: "materialized-budget", + table: &table, + store: &boxed_store, + backend: BackendLabel::Sim, + initial_fields: &serde_json::json!({}), + blob_store: None, + }, + true, + ) + .await + .expect("the admitted raw journal tail must remain restartable"); + + assert_eq!(recovered.state.sequence_nr, 10_000); + assert_eq!(recovered.state.events_since_snapshot, 10_000); + assert_eq!(recovered.state.total_event_count, 9_999); + assert!(!recovered.state.can_accept_event()); +} + +#[cfg(feature = "sim")] +#[tokio::test] +async fn snapshot_ahead_of_journal_is_not_claimed_as_a_lower_applied_snapshot() { + use temper_runtime::persistence::EventStore; + use temper_store_sim::SimEventStore; + + let store = SimEventStore::no_faults(284); + let boxed_store = crate::storage::BoxedEventStore::new(store.clone()); + let persistence_id = "default:Order:snapshot-ahead"; + let durable_snapshot = b"snapshot-5".to_vec(); + store + .save_snapshot(persistence_id, 5, &durable_snapshot) + .await + .expect("seed migration snapshot"); + let mut state = EntityState { + entity_type: "Order".to_string(), + entity_id: "snapshot-ahead".to_string(), + status: "Draft".to_string(), + item_count: 0, + counters: BTreeMap::new(), + booleans: BTreeMap::new(), + lists: BTreeMap::new(), + fields: serde_json::json!({"Id": "snapshot-ahead", "Status": "Draft"}), + events: std::collections::VecDeque::new(), + total_event_count: 2, + events_since_snapshot: 2, + last_snapshot_sequence_nr: 0, + sequence_nr: 2, + processed_idempotency_keys: BTreeMap::new(), + }; + let mut source = temper_runtime::persistence::SnapshotSourceFence::Exact { + sequence_nr: 5, + state: durable_snapshot.clone(), + }; + + let attempted = EntityActor::maybe_save_snapshot( + &boxed_store, + None, + persistence_id, + &mut state, + &mut source, + None, + ) + .await + .expect("skip lower journal-aligned snapshot"); + + assert_eq!(attempted, None); + assert_eq!(state.last_snapshot_sequence_nr, 0); + assert_eq!(state.events_since_snapshot, 2); + assert_eq!( + store.load_snapshot(persistence_id).await.unwrap(), + Some((5, durable_snapshot)) + ); +} + +#[cfg(feature = "sim")] +#[tokio::test] +async fn stable_recovery_preserves_historical_snapshot_nondeterministic_effect_values() { + use temper_runtime::persistence::EventStore; + use temper_runtime::scheduler::install_deterministic_context; + use temper_store_sim::SimEventStore; + + let (_guard, _clock, _ids) = install_deterministic_context(405); + let table = TransitionTable::from_ioa_source( + r#" +[automaton] +name = "Parent" +states = ["Idle", "Active"] +initial = "Idle" + +[[action]] +name = "Start" +kind = "input" +from = ["Idle"] +to = "Active" +params = [] +effect = [{ type = "spawn", entity_type = "Child", entity_id_source = "{uuid}", initial_action = "Begin", store_id_in = "last_child_id" }] +"#, + ); + let store = SimEventStore::no_faults(405); + let boxed_store = crate::storage::BoxedEventStore::new(store.clone()); + let persistence_id = "default:Parent:journal-snapshot"; + let event = EntityEvent { + action: "Start".to_string(), + from_status: "Idle".to_string(), + to_status: "Active".to_string(), + timestamp: sim_now(), + params: serde_json::json!({}), + idempotency_key: Some("start-once".to_string()), + }; + store + .append( + persistence_id, + 0, + &[PersistenceEnvelope { + sequence_nr: 0, + event_type: event.action.clone(), + payload: serde_json::to_value(&event).expect("encode Start event"), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: event.timestamp, + actor_id: persistence_id.to_string(), + }, + }], + ) + .await + .expect("seed Start journal event"); + let snapshot_state = EntityState { + entity_type: "Parent".to_string(), + entity_id: "journal-snapshot".to_string(), + status: "Active".to_string(), + item_count: 0, + counters: BTreeMap::new(), + booleans: BTreeMap::new(), + lists: BTreeMap::new(), + fields: serde_json::json!({ + "Id": "journal-snapshot", + "Status": "Active", + "last_child_id": "durable-child-id" + }), + events: std::collections::VecDeque::new(), + total_event_count: 1, + events_since_snapshot: 0, + last_snapshot_sequence_nr: 1, + sequence_nr: 1, + processed_idempotency_keys: BTreeMap::from([("start-once".to_string(), 1)]), + }; + let mut legacy_snapshot = + serde_json::to_value(&snapshot_state).expect("encode pre-coordinate actor snapshot"); + let legacy_snapshot = legacy_snapshot + .as_object_mut() + .expect("actor snapshot must be an object"); + legacy_snapshot.remove("events"); + legacy_snapshot.remove("events_since_snapshot"); + legacy_snapshot.remove("last_snapshot_sequence_nr"); + let snapshot = + serde_json::to_vec(&legacy_snapshot).expect("serialize pre-coordinate actor snapshot"); + store + .save_snapshot(persistence_id, 1, &snapshot) + .await + .expect("seed journal-aligned snapshot"); + + let recovered = recover_entity_state_from_stable_sources(EntityRecoveryContext { + tenant: "default", + entity_type: "Parent", + entity_id: "journal-snapshot", + table: &table, + store: &boxed_store, + backend: BackendLabel::Sim, + initial_fields: &serde_json::json!({}), + blob_store: None, + }) + .await + .expect("stable recovery should use the journal-aligned snapshot boundary"); + + assert_eq!( + recovered + .state + .expect("journal-backed state") + .fields + .get("last_child_id"), + Some(&serde_json::json!("durable-child-id")), + "replaying the snapshotted Start event must not generate a replacement child id" + ); + + let passivation_id = "passivation-snapshot"; + let passivation_persistence_id = "default:Parent:passivation-snapshot"; + let passivation_event = EntityEvent { + action: "Start".to_string(), + from_status: "Idle".to_string(), + to_status: "Active".to_string(), + timestamp: sim_now(), + params: serde_json::json!({}), + idempotency_key: Some("passivation-start-once".to_string()), + }; + store + .append( + passivation_persistence_id, + 0, + &[PersistenceEnvelope { + sequence_nr: 0, + event_type: passivation_event.action.clone(), + payload: serde_json::to_value(&passivation_event) + .expect("encode passivation Start event"), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: passivation_event.timestamp, + actor_id: passivation_persistence_id.to_string(), + }, + }], + ) + .await + .expect("seed passivation Start journal event"); + let mut passivation_state = snapshot_state.clone(); + passivation_state.entity_id = passivation_id.to_string(); + passivation_state.fields = serde_json::json!({ + "Id": passivation_id, + "Status": "Active", + "last_child_id": "passivation-child-id" + }); + passivation_state.events_since_snapshot = 1; + passivation_state.last_snapshot_sequence_nr = 0; + passivation_state.processed_idempotency_keys = + BTreeMap::from([("passivation-start-once".to_string(), 1)]); + let mut passivation_snapshot = + serde_json::to_value(&passivation_state).expect("encode historical passivation snapshot"); + passivation_snapshot + .as_object_mut() + .expect("passivation snapshot must be an object") + .remove("events"); + store + .save_snapshot( + passivation_persistence_id, + 1, + &serde_json::to_vec(&passivation_snapshot) + .expect("serialize historical passivation snapshot"), + ) + .await + .expect("seed historical passivation snapshot"); + + let recovered = recover_entity_state_from_stable_sources(EntityRecoveryContext { + tenant: "default", + entity_type: "Parent", + entity_id: passivation_id, + table: &table, + store: &boxed_store, + backend: BackendLabel::Sim, + initial_fields: &serde_json::json!({}), + blob_store: None, + }) + .await + .expect("stable recovery should use the historical passivation boundary"); + assert_eq!( + recovered + .state + .expect("passivation journal-backed state") + .fields + .get("last_child_id"), + Some(&serde_json::json!("passivation-child-id")), + "historical passivation recovery must not generate a replacement child id" + ); +} + +#[test] +fn legacy_snapshot_provenance_requires_an_actor_writer_shape() { + let snapshot_state = EntityState { + entity_type: "Parent".to_string(), + entity_id: "legacy-shape".to_string(), + status: "Active".to_string(), + item_count: 0, + counters: BTreeMap::new(), + booleans: BTreeMap::new(), + lists: BTreeMap::new(), + fields: serde_json::json!({"Id": "legacy-shape", "Status": "Active"}), + events: std::collections::VecDeque::new(), + total_event_count: 1, + events_since_snapshot: 0, + last_snapshot_sequence_nr: 1, + sequence_nr: 1, + processed_idempotency_keys: BTreeMap::new(), + }; + let segmented = EntityActor::serialize_snapshot_state(&snapshot_state, None) + .expect("encode segmented pre-provenance snapshot"); + let mut restored = snapshot_state.clone(); + assert_eq!( + EntityActor::apply_snapshot_bytes(&mut restored, 1, &segmented), + Some(SnapshotProvenance::LegacyJournal { + through_sequence: 1 + }) + ); + + let mut passivation = + serde_json::to_value(&snapshot_state).expect("encode historical passivation snapshot"); + let passivation = passivation + .as_object_mut() + .expect("passivation snapshot must be an object"); + passivation.remove("events"); + passivation.insert("events_since_snapshot".to_string(), serde_json::json!(1)); + passivation.insert( + "last_snapshot_sequence_nr".to_string(), + serde_json::json!(0), + ); + let passivation = + serde_json::to_vec(&passivation).expect("serialize historical passivation snapshot"); + assert_eq!( + EntityActor::apply_snapshot_bytes(&mut restored, 1, &passivation), + Some(SnapshotProvenance::LegacyJournal { + through_sequence: 1 + }) + ); + + let mut pre_coordinate = + serde_json::to_value(&snapshot_state).expect("encode pre-coordinate actor snapshot"); + let pre_coordinate = pre_coordinate + .as_object_mut() + .expect("actor snapshot must be an object"); + pre_coordinate.remove("events"); + pre_coordinate.remove("events_since_snapshot"); + pre_coordinate.remove("last_snapshot_sequence_nr"); + let pre_coordinate = + serde_json::to_vec(&pre_coordinate).expect("serialize pre-coordinate actor snapshot"); + assert_eq!( + EntityActor::apply_snapshot_bytes(&mut restored, 1, &pre_coordinate), + Some(SnapshotProvenance::LegacyJournal { + through_sequence: 1 + }) + ); + + let generic_migration = + serde_json::to_vec(&snapshot_state).expect("encode coordinate-bearing migration"); + assert_eq!( + EntityActor::apply_snapshot_bytes(&mut restored, 1, &generic_migration), + Some(SnapshotProvenance::Legacy) + ); +} + +#[cfg(feature = "sim")] +#[tokio::test] +async fn actor_recovery_rejects_snapshot_identity_mismatch() { + use temper_runtime::persistence::EventStore; + use temper_store_sim::SimEventStore; + + let store = SimEventStore::no_faults(285); + let boxed_store = crate::storage::BoxedEventStore::new(store.clone()); + let persistence_id = "default:Order:expected-id"; + let mismatched = serde_json::to_vec(&serde_json::json!({ + "entity_type": "Order", + "entity_id": "different-id", + "status": "Draft", + "item_count": 0, + "fields": {"Id": "different-id", "Status": "Draft"} + })) + .expect("serialize mismatched snapshot"); + store + .save_snapshot(persistence_id, 5, &mismatched) + .await + .expect("seed mismatched snapshot"); + let table = order_table().read().expect("table lock").clone(); + let result = recover_entity_state_with_source_from_store( + EntityRecoveryContext { + tenant: "default", + entity_type: "Order", + entity_id: "expected-id", + table: &table, + store: &boxed_store, + backend: BackendLabel::Sim, + initial_fields: &serde_json::json!({}), + blob_store: None, + }, + true, + ) + .await; + let error = match result { + Ok(_) => panic!("mismatched snapshot identity must fail closed"), + Err(error) => error, + }; + assert!( + error.to_string().contains("snapshot identity mismatch"), + "unexpected recovery error: {error}" + ); +} + +#[cfg(feature = "sim")] +#[tokio::test] +async fn actor_recovery_rejects_snapshot_status_outside_the_transition_table() { + use temper_runtime::persistence::EventStore; + use temper_store_sim::SimEventStore; + + let store = SimEventStore::no_faults(289); + let boxed_store = crate::storage::BoxedEventStore::new(store.clone()); + let persistence_id = "default:Order:invalid-status"; + let snapshot = serde_json::to_vec(&serde_json::json!({ + "entity_type": "Order", + "entity_id": "invalid-status", + "status": "NotADeclaredState", + "item_count": 0, + "fields": {"Id": "invalid-status", "Status": "NotADeclaredState"} + })) + .expect("serialize invalid-status snapshot"); + store + .save_snapshot(persistence_id, 5, &snapshot) + .await + .expect("seed invalid-status snapshot"); + let table = order_table().read().expect("table lock").clone(); + let result = recover_entity_state_with_source_from_store( + EntityRecoveryContext { + tenant: "default", + entity_type: "Order", + entity_id: "invalid-status", + table: &table, + store: &boxed_store, + backend: BackendLabel::Sim, + initial_fields: &serde_json::json!({}), + blob_store: None, + }, + true, + ) + .await; + let error = match result { + Ok(_) => panic!("snapshot status outside the transition table must fail closed"), + Err(error) => error, + }; + assert!( + error.to_string().contains("snapshot has invalid status"), + "unexpected recovery error: {error}" + ); +} + +#[cfg(feature = "sim")] +#[tokio::test] +async fn domain_action_named_like_materialization_replays_as_a_domain_event() { + use temper_runtime::persistence::EventStore; + use temper_store_sim::SimEventStore; + + let store = SimEventStore::no_faults(290); + let boxed_store = crate::storage::BoxedEventStore::new(store.clone()); + let persistence_id = "default:Collision:domain-action"; + let table = TransitionTable::from_ioa_source( + r#" +[automaton] +name = "Collision" +states = ["Draft", "Ready"] +initial = "Draft" + +[[action]] +name = "Temper.Internal.StateMaterialization.v1" +kind = "input" +from = ["Draft"] +to = "Ready" +"#, + ); + let timestamp = chrono::DateTime::UNIX_EPOCH; + store + .append( + persistence_id, + 0, + &[PersistenceEnvelope { + sequence_nr: 0, + event_type: STATE_MATERIALIZATION_EVENT_TYPE.to_string(), + payload: serde_json::to_value(EntityEvent { + action: STATE_MATERIALIZATION_EVENT_TYPE.to_string(), + from_status: "Draft".to_string(), + to_status: "Ready".to_string(), + timestamp, + params: serde_json::json!({}), + idempotency_key: None, + }) + .expect("serialize domain collision event"), + metadata: EventMetadata { + event_id: uuid::Uuid::nil(), + causation_id: uuid::Uuid::nil(), + correlation_id: uuid::Uuid::nil(), + timestamp, + actor_id: persistence_id.to_string(), + }, + }], + ) + .await + .expect("append legal domain action collision"); + + let recovered = recover_entity_state_with_source_from_store( + EntityRecoveryContext { + tenant: "default", + entity_type: "Collision", + entity_id: "domain-action", + table: &table, + store: &boxed_store, + backend: BackendLabel::Sim, + initial_fields: &serde_json::json!({}), + blob_store: None, + }, + true, + ) + .await + .expect("domain collision must replay normally"); + assert_eq!(recovered.state.status, "Ready"); + assert_eq!(recovered.state.sequence_nr, 1); +} + +#[cfg(feature = "sim")] +#[tokio::test] +async fn actor_recovery_rejects_out_of_position_state_materialization() { + use temper_runtime::persistence::EventStore; + use temper_store_sim::SimEventStore; + + let store = SimEventStore::no_faults(286); + let boxed_store = crate::storage::BoxedEventStore::new(store.clone()); + let persistence_id = "default:Order:late-materialization"; + let metadata = || EventMetadata { + event_id: uuid::Uuid::nil(), + causation_id: uuid::Uuid::nil(), + correlation_id: uuid::Uuid::nil(), + timestamp: chrono::DateTime::UNIX_EPOCH, + actor_id: persistence_id.to_string(), + }; + let baseline = EntityState { + entity_type: "Order".to_string(), + entity_id: "late-materialization".to_string(), + status: "Draft".to_string(), + item_count: 9, + counters: BTreeMap::new(), + booleans: BTreeMap::new(), + lists: BTreeMap::new(), + fields: serde_json::json!({"Id": "late-materialization", "Status": "Draft"}), + events: std::collections::VecDeque::new(), + total_event_count: 9, + events_since_snapshot: 0, + last_snapshot_sequence_nr: 0, + sequence_nr: 0, + processed_idempotency_keys: BTreeMap::new(), + }; + store + .append( + persistence_id, + 0, + &[ + PersistenceEnvelope { + sequence_nr: 0, + event_type: "Created".to_string(), + payload: serde_json::to_value(EntityEvent { + action: "Created".to_string(), + from_status: String::new(), + to_status: "Draft".to_string(), + timestamp: chrono::DateTime::UNIX_EPOCH, + params: serde_json::json!({}), + idempotency_key: None, + }) + .unwrap(), + metadata: metadata(), + }, + PersistenceEnvelope { + sequence_nr: 0, + event_type: STATE_MATERIALIZATION_EVENT_TYPE.to_string(), + payload: serde_json::to_value(PersistedStateMaterialization { + schema: STATE_MATERIALIZATION_SCHEMA.to_string(), + state: baseline, + }) + .unwrap(), + metadata: metadata(), + }, + ], + ) + .await + .expect("seed out-of-position materialization"); + let table = order_table().read().expect("table lock").clone(); + let result = recover_entity_state_with_source_from_store( + EntityRecoveryContext { + tenant: "default", + entity_type: "Order", + entity_id: "late-materialization", + table: &table, + store: &boxed_store, + backend: BackendLabel::Sim, + initial_fields: &serde_json::json!({}), + blob_store: None, + }, + true, + ) + .await; + let error = match result { + Ok(_) => panic!("late state materialization must fail closed"), + Err(error) => error, + }; + assert!( + error + .to_string() + .contains("must be the first journal event"), + "unexpected recovery error: {error}" + ); +} + #[test] fn event_budget_workspace_id_uses_workspace_entity_id_or_field() { let workspace_state = EntityState { @@ -107,12 +891,15 @@ async fn queued_snapshot_only_advances_replay_boundary_after_write_applies() { sequence_nr: 100, processed_idempotency_keys: BTreeMap::new(), }; + let mut snapshot_source = temper_runtime::persistence::SnapshotSourceFence::Absent; EntityActor::maybe_save_snapshot( &boxed_store, Some(&snapshot_queue), persistence_id, &mut state, + &mut snapshot_source, + None, ) .await .expect("snapshot enqueue should succeed"); @@ -137,12 +924,213 @@ async fn queued_snapshot_only_advances_replay_boundary_after_write_applies() { Some(&snapshot_queue), persistence_id, &mut state, + &mut snapshot_source, + None, ) .await .expect("snapshot boundary observation should succeed"); assert_eq!(state.last_snapshot_sequence_nr, 100); assert_eq!(state.events_since_snapshot, 1); + assert!(matches!( + snapshot_source, + temper_runtime::persistence::SnapshotSourceFence::Exact { + sequence_nr: 100, + .. + } + )); +} + +#[cfg(feature = "sim")] +#[tokio::test] +async fn queued_snapshot_carries_the_applied_exact_source_into_the_next_write() { + use temper_runtime::persistence::{EventStore, SnapshotSourceFence}; + use temper_store_sim::SimEventStore; + + let store = SimEventStore::no_faults(44); + let boxed_store = crate::storage::BoxedEventStore::new(store.clone()); + let snapshot_queue = SnapshotWriteQueue::start(boxed_store.clone()); + let persistence_id = "default:Order:queued-snapshot-chain"; + let mut state = EntityState { + entity_type: "Order".to_string(), + entity_id: "queued-snapshot-chain".to_string(), + status: "Draft".to_string(), + item_count: 0, + counters: BTreeMap::new(), + booleans: BTreeMap::new(), + lists: BTreeMap::new(), + fields: serde_json::json!({ + "Id": "queued-snapshot-chain", + "Status": "Draft" + }), + events: std::collections::VecDeque::new(), + total_event_count: 100, + events_since_snapshot: 100, + last_snapshot_sequence_nr: 0, + sequence_nr: 100, + processed_idempotency_keys: BTreeMap::new(), + }; + let mut source = SnapshotSourceFence::Absent; + + EntityActor::maybe_save_snapshot( + &boxed_store, + Some(&snapshot_queue), + persistence_id, + &mut state, + &mut source, + None, + ) + .await + .expect("enqueue first snapshot"); + for _ in 0..20 { + if snapshot_queue.applied_sequence(persistence_id) == Some(100) { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert_eq!(snapshot_queue.applied_sequence(persistence_id), Some(100)); + + state.sequence_nr = 200; + state.total_event_count = 200; + state.events_since_snapshot = 200; + EntityActor::maybe_save_snapshot( + &boxed_store, + Some(&snapshot_queue), + persistence_id, + &mut state, + &mut source, + None, + ) + .await + .expect("enqueue second snapshot from the same actor source"); + for _ in 0..20 { + if snapshot_queue.applied_sequence(persistence_id) == Some(200) { + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + + assert_eq!( + snapshot_queue.applied_sequence(persistence_id), + Some(200), + "the second queued write must fence against the exact first snapshot" + ); + assert_eq!( + store + .load_snapshot(persistence_id) + .await + .expect("load chained snapshot") + .map(|(sequence_nr, _)| sequence_nr), + Some(200) + ); +} + +#[cfg(feature = "sim")] +#[tokio::test] +async fn passivation_snapshot_keeps_the_contract_that_produced_actor_state() { + use temper_runtime::persistence::{ + EventStore, PersistenceError, decode_activated_key_contract, + }; + use temper_store_sim::SimEventStore; + + const IOA: &str = r#" +[automaton] +name = "Doc" +states = ["New", "Ready"] +initial = "New" + +[[state]] +name = "Path" +type = "string" +initial = "" + +[[key]] +name = "path" +properties = ["Path"] + +[[action]] +name = "Create" +kind = "input" +from = ["New"] +to = "Ready" +params = ["Path"] +"#; + + let store = SimEventStore::no_faults(297); + let boxed = crate::storage::BoxedEventStore::new(store.clone()); + let mut epoch_one_table = TransitionTable::from_ioa_source(IOA); + let signature = crate::key_index::declared_key_set_signature(&epoch_one_table.keys); + let epoch_one = store + .activate_key_index_contract("default", "Doc", &signature, false) + .await + .expect("activate epoch one"); + store + .mark_key_index_backfilled("default", "Doc", &signature) + .await + .expect("publish epoch-one readiness"); + epoch_one_table.key_contract_activation_epoch = epoch_one; + let shared_table = Arc::new(RwLock::new(epoch_one_table)); + let system = ActorSystem::new("passivation-contract-provenance"); + let actor = system.spawn( + EntityActor::with_persistence( + "Doc", + "provenance", + shared_table.clone(), + serde_json::json!({}), + boxed.clone(), + crate::storage::BackendLabel::Sim, + ), + "provenance", + ); + let created: EntityResponse = actor + .ask( + EntityMsg::Action { + name: "Create".to_string(), + params: serde_json::json!({"Path": "/owned"}), + cross_entity_booleans: BTreeMap::new(), + idempotency_key: None, + }, + Duration::from_secs(1), + ) + .await + .expect("Create response"); + assert!(created.success, "Create failed: {:?}", created.error); + + let epoch_two = store + .activate_key_index_contract("default", "Doc", &signature, false) + .await + .expect("activate epoch two"); + let mut epoch_two_table = TransitionTable::from_ioa_source(IOA); + epoch_two_table.key_contract_activation_epoch = epoch_two; + *shared_table.write().expect("table lock") = epoch_two_table; + + let passivation: super::super::types::EntityPassivationSnapshot = actor + .ask(EntityMsg::GetPassivationSnapshot, Duration::from_secs(1)) + .await + .expect("passivation response"); + assert_eq!( + decode_activated_key_contract(&passivation.key_contract).1, + Some(epoch_one), + "a live table swap must not rewrite actor-state provenance" + ); + let snapshot = serde_json::to_vec(&passivation.state).expect("encode test snapshot"); + let error = boxed + .save_snapshot_if_source( + "default:Doc:provenance", + passivation.state.sequence_nr, + &snapshot, + &passivation.snapshot_source, + Some(&passivation.key_contract), + ) + .await + .expect_err("epoch-one passivation snapshot must be stale under epoch two"); + assert!(matches!( + error, + PersistenceError::KeyContractActivationStale { + activated_epoch, + attempted_epoch: Some(attempted_epoch), + } if activated_epoch == epoch_two && attempted_epoch == epoch_one + )); } // ============================================= @@ -469,14 +1457,15 @@ async fn dst_multiple_actors_independent() { assert_eq!(r2.state.item_count, 1); } -/// Verify that replay skips events whose payload cannot be deserialized against -/// the current `EntityEvent` schema (schema evolution resilience). +/// Verify that replay fails closed when a committed event cannot be decoded +/// against the current `EntityEvent` schema. /// -/// The actor must reach a consistent final state using only the events that -/// parsed successfully, and must NOT panic on the schema-mismatched event. +/// Skipping the malformed event would silently construct state from only a +/// prefix of the durable history. The actor must stop before serving that +/// partial state. #[cfg(feature = "sim")] #[tokio::test] -async fn replay_skips_schema_mismatched_events() { +async fn replay_rejects_schema_mismatched_events_before_serving_state() { use temper_runtime::persistence::EventStore; use temper_store_sim::SimEventStore; @@ -535,19 +1524,12 @@ async fn replay_skips_schema_mismatched_events() { ); let actor_ref = system.spawn(actor, "schema-evo-1"); - let response: EntityResponse = actor_ref - .ask(EntityMsg::GetState, Duration::from_secs(5)) + let error = actor_ref + .ask::(EntityMsg::GetState, Duration::from_secs(5)) .await - .unwrap(); + .expect_err("malformed durable history must stop the actor"); - // Actor started cleanly despite the bad event. - assert!(response.success); - // The valid CancelOrder event was applied → status is Cancelled. - assert_eq!(response.state.status, "Cancelled"); - // Both sequence numbers consumed (bad event's seq_nr was still advanced). - assert_eq!(response.state.sequence_nr, 2); - // Only the good event contributed to total_event_count. - assert_eq!(response.state.total_event_count, 1); + assert_eq!(error, temper_runtime::actor::ActorError::Stopped); } /// A committed cross-entity-guarded transition must survive replay. diff --git a/crates/temper-server/src/entity_actor/mod.rs b/crates/temper-server/src/entity_actor/mod.rs index a0f31f33c..0cc393202 100644 --- a/crates/temper-server/src/entity_actor/mod.rs +++ b/crates/temper-server/src/entity_actor/mod.rs @@ -11,11 +11,18 @@ mod snapshot_queue; pub mod types; pub use actor::EntityActor; +#[cfg(test)] pub(crate) use actor::recover_entity_state_from_store; +pub(crate) use actor::{ + CapturedEntitySnapshot, EntityRecoveryContext, StableEntitySource, + recover_entity_state_from_stable_sources, stable_entity_source_is_current, + state_materialization_envelope, +}; pub use effects::{ ProcessResult, ScheduledAction, apply_effects, apply_new_state_fallback, build_eval_context, process_action, process_action_with_xref, sync_fields, }; pub use sim_handler::EntityActorHandler; pub(crate) use snapshot_queue::SnapshotWriteQueue; +pub(crate) use types::EntityPassivationSnapshot; pub use types::{EntityEvent, EntityMsg, EntityResponse, EntityState}; diff --git a/crates/temper-server/src/entity_actor/snapshot_queue.rs b/crates/temper-server/src/entity_actor/snapshot_queue.rs index aadffd0ae..3c312d037 100644 --- a/crates/temper-server/src/entity_actor/snapshot_queue.rs +++ b/crates/temper-server/src/entity_actor/snapshot_queue.rs @@ -4,10 +4,12 @@ //! recovery accelerators, so this queue moves their writes off the actor hot //! path while preserving the latest accepted sequence per persistence stream. +use sha2::{Digest, Sha256}; use std::collections::BTreeMap; use std::sync::{Arc, Mutex}; use std::time::{Duration, Instant}; +use temper_runtime::persistence::{PersistenceError, SnapshotSourceFence}; use tokio::sync::Notify; use tracing::Instrument; @@ -21,9 +23,41 @@ struct QueuedSnapshotWrite { persistence_id: String, sequence_nr: u64, snapshot: Vec, + source: SnapshotSourceFence, + key_contract: Option, enqueued_at: Instant, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct SnapshotWriteBoundary { + activation_epoch: u64, + sequence_nr: u64, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +struct AppliedSnapshotWrite { + boundary: SnapshotWriteBoundary, + snapshot_sha256: [u8; 32], +} + +fn activation_epoch(key_contract: Option<&str>) -> u64 { + key_contract + .and_then(|contract| temper_runtime::persistence::decode_activated_key_contract(contract).1) + .unwrap_or(0) +} + +fn boundary_precedes( + existing_contract: Option<&str>, + existing_sequence: u64, + incoming_contract: Option<&str>, + incoming_sequence: u64, +) -> bool { + let existing_epoch = activation_epoch(existing_contract); + let incoming_epoch = activation_epoch(incoming_contract); + existing_epoch < incoming_epoch + || (existing_epoch == incoming_epoch && existing_sequence < incoming_sequence) +} + #[derive(Debug, Default)] struct PendingSnapshotWrites { writes: BTreeMap, @@ -41,7 +75,7 @@ pub(crate) enum SnapshotEnqueueOutcome { pub(crate) struct SnapshotWriteQueue { store: BoxedEventStore, pending: Arc>, - applied_sequences: Arc>>, + applied_snapshots: Arc>>, notify: Arc, capacity: usize, drain_batch: usize, @@ -52,7 +86,7 @@ impl SnapshotWriteQueue { let queue = Arc::new(Self { store, pending: Arc::new(Mutex::new(PendingSnapshotWrites::default())), - applied_sequences: Arc::new(Mutex::new(BTreeMap::new())), + applied_snapshots: Arc::new(Mutex::new(BTreeMap::new())), notify: Arc::new(Notify::new()), capacity: snapshot_queue_capacity(), drain_batch: snapshot_drain_batch(), @@ -66,7 +100,7 @@ impl SnapshotWriteQueue { Self { store, pending: Arc::new(Mutex::new(PendingSnapshotWrites::default())), - applied_sequences: Arc::new(Mutex::new(BTreeMap::new())), + applied_snapshots: Arc::new(Mutex::new(BTreeMap::new())), notify: Arc::new(Notify::new()), capacity, drain_batch, @@ -78,10 +112,29 @@ impl SnapshotWriteQueue { persistence_id: String, sequence_nr: u64, snapshot: Vec, + source: SnapshotSourceFence, + key_contract: Option, ) -> SnapshotEnqueueOutcome { + if matches!( + &source, + SnapshotSourceFence::Exact { + sequence_nr: source_sequence, + .. + } if *source_sequence > sequence_nr + ) { + crate::runtime_metrics::record_snapshot_write_stale_skipped(); + return SnapshotEnqueueOutcome::StaleSkipped; + } let mut pending = self.pending.lock().expect("snapshot queue mutex poisoned"); let existing = pending.writes.get(&persistence_id); - if existing.is_some_and(|existing| existing.sequence_nr >= sequence_nr) { + if existing.is_some_and(|existing| { + !boundary_precedes( + existing.key_contract.as_deref(), + existing.sequence_nr, + key_contract.as_deref(), + sequence_nr, + ) + }) { crate::runtime_metrics::record_snapshot_write_stale_skipped(); return SnapshotEnqueueOutcome::StaleSkipped; } @@ -106,6 +159,8 @@ impl SnapshotWriteQueue { persistence_id, sequence_nr, snapshot, + source, + key_contract, enqueued_at: Instant::now(), }, ); @@ -116,14 +171,51 @@ impl SnapshotWriteQueue { outcome } + #[cfg(test)] pub(crate) fn applied_sequence(&self, persistence_id: &str) -> Option { - self.applied_sequences + self.applied_snapshots + .lock() + .expect("snapshot applied sequence mutex poisoned") + .get(persistence_id) + .map(|applied| applied.boundary.sequence_nr) + } + + pub(crate) fn applied_sequence_for_contract( + &self, + persistence_id: &str, + key_contract: Option<&str>, + ) -> Option { + let incoming_epoch = activation_epoch(key_contract); + self.applied_snapshots .lock() .expect("snapshot applied sequence mutex poisoned") .get(persistence_id) - .copied() + .filter(|applied| applied.boundary.activation_epoch >= incoming_epoch) + .map(|applied| applied.boundary.sequence_nr) + } + + pub(crate) async fn applied_source_for_contract( + &self, + persistence_id: &str, + key_contract: Option<&str>, + ) -> Option { + let incoming_epoch = activation_epoch(key_contract); + let applied = self + .applied_snapshots + .lock() + .expect("snapshot applied source mutex poisoned") + .get(persistence_id) + .filter(|applied| applied.boundary.activation_epoch == incoming_epoch) + .cloned()?; + let (sequence_nr, state) = self.store.load_snapshot(persistence_id).await.ok()??; + let digest: [u8; 32] = Sha256::digest(&state).into(); + if sequence_nr != applied.boundary.sequence_nr || digest != applied.snapshot_sha256 { + return None; + } + Some(SnapshotSourceFence::Exact { sequence_nr, state }) } + #[cfg(test)] pub(crate) fn pending_sequence(&self, persistence_id: &str) -> Option { self.pending .lock() @@ -133,6 +225,21 @@ impl SnapshotWriteQueue { .map(|write| write.sequence_nr) } + pub(crate) fn pending_sequence_for_contract( + &self, + persistence_id: &str, + key_contract: Option<&str>, + ) -> Option { + let incoming_epoch = activation_epoch(key_contract); + self.pending + .lock() + .expect("snapshot queue mutex poisoned") + .writes + .get(persistence_id) + .filter(|write| activation_epoch(write.key_contract.as_deref()) >= incoming_epoch) + .map(|write| write.sequence_nr) + } + fn spawn_worker(self: &Arc) { let queue = Arc::clone(self); tokio::spawn(async move { @@ -167,7 +274,7 @@ impl SnapshotWriteQueue { writes } - async fn apply(&self, write: QueuedSnapshotWrite) { + async fn apply(&self, mut write: QueuedSnapshotWrite) { let span = tracing::info_span!( "dispatch.phase.snapshot.queued", otel.name = "dispatch.phase.snapshot.queued", @@ -181,9 +288,38 @@ impl SnapshotWriteQueue { crate::runtime_metrics::record_snapshot_write_queue_wait( started_at.duration_since(write.enqueued_at), ); + if let Some(applied_source) = self + .applied_source_for_contract(&write.persistence_id, write.key_contract.as_deref()) + .await + { + let SnapshotSourceFence::Exact { + sequence_nr: applied_sequence, + .. + } = &applied_source + else { + unreachable!("an applied snapshot source is always exact"); + }; + if *applied_sequence > write.sequence_nr { + crate::runtime_metrics::record_snapshot_write_stale_skipped(); + return; + } + let supplied_sequence = match &write.source { + SnapshotSourceFence::Exact { sequence_nr, .. } => Some(*sequence_nr), + SnapshotSourceFence::Absent | SnapshotSourceFence::Unchecked => None, + }; + if supplied_sequence.is_none_or(|sequence_nr| *applied_sequence >= sequence_nr) { + write.source = applied_source; + } + } let result = self .store - .save_snapshot(&write.persistence_id, write.sequence_nr, &write.snapshot) + .save_snapshot_if_source( + &write.persistence_id, + write.sequence_nr, + &write.snapshot, + &write.source, + write.key_contract.as_deref(), + ) .await; let result_label = if result.is_ok() { "ok" } else { "error" }; crate::runtime_metrics::record_snapshot_write_duration( @@ -200,7 +336,31 @@ impl SnapshotWriteQueue { crate::runtime_metrics::record_snapshot_write_applied_sequence( write.sequence_nr, ); - self.record_applied_sequence(&write.persistence_id, write.sequence_nr); + self.record_applied_snapshot( + &write.persistence_id, + write.sequence_nr, + &write.snapshot, + write.key_contract.as_deref(), + ); + } + Err(PersistenceError::SnapshotGenerationChanged) => { + crate::runtime_metrics::record_snapshot_write_stale_skipped(); + tracing::warn!( + persistence_id = %write.persistence_id, + sequence_nr = write.sequence_nr, + "discarding queued snapshot because its source generation changed" + ); + } + Err( + PersistenceError::KeyContractNotActive { .. } + | PersistenceError::KeyContractActivationStale { .. }, + ) => { + crate::runtime_metrics::record_snapshot_write_stale_skipped(); + tracing::warn!( + persistence_id = %write.persistence_id, + sequence_nr = write.sequence_nr, + "discarding queued snapshot because its key contract is stale" + ); } Err(e) => { crate::runtime_metrics::record_snapshot_write_error(); @@ -219,23 +379,58 @@ impl SnapshotWriteQueue { .await; } - fn record_applied_sequence(&self, persistence_id: &str, sequence_nr: u64) { + fn record_applied_snapshot( + &self, + persistence_id: &str, + sequence_nr: u64, + snapshot: &[u8], + key_contract: Option<&str>, + ) { let mut applied = self - .applied_sequences + .applied_snapshots .lock() .expect("snapshot applied sequence mutex poisoned"); - let entry = applied.entry(persistence_id.to_string()).or_insert(0); - if sequence_nr > *entry { - *entry = sequence_nr; + let incoming = AppliedSnapshotWrite { + boundary: SnapshotWriteBoundary { + activation_epoch: activation_epoch(key_contract), + sequence_nr, + }, + snapshot_sha256: Sha256::digest(snapshot).into(), + }; + let entry = applied + .entry(persistence_id.to_string()) + .or_insert_with(|| incoming.clone()); + if entry.boundary.activation_epoch < incoming.boundary.activation_epoch + || (entry.boundary.activation_epoch == incoming.boundary.activation_epoch + && entry.boundary.sequence_nr < incoming.boundary.sequence_nr) + { + *entry = incoming; } } + #[cfg(test)] + fn record_applied_sequence( + &self, + persistence_id: &str, + sequence_nr: u64, + key_contract: Option<&str>, + ) { + self.record_applied_snapshot(persistence_id, sequence_nr, &[], key_contract); + } + fn requeue_failed_write(&self, write: QueuedSnapshotWrite) { let mut pending = self.pending.lock().expect("snapshot queue mutex poisoned"); if pending .writes .get(&write.persistence_id) - .is_some_and(|existing| existing.sequence_nr >= write.sequence_nr) + .is_some_and(|existing| { + !boundary_precedes( + existing.key_contract.as_deref(), + existing.sequence_nr, + write.key_contract.as_deref(), + write.sequence_nr, + ) + }) { crate::runtime_metrics::record_snapshot_write_queue_depth(pending.writes.len() as u64); return; @@ -264,143 +459,5 @@ fn snapshot_drain_batch() -> usize { } #[cfg(test)] -mod tests { - use super::*; - use std::sync::atomic::{AtomicUsize, Ordering}; - use temper_runtime::persistence::{ - EventStore, PersistenceAppend, PersistenceAppendResult, PersistenceEnvelope, - PersistenceError, - }; - - #[derive(Default)] - struct RecordingEventStore { - saves: AtomicUsize, - } - - impl EventStore for RecordingEventStore { - async fn append( - &self, - _persistence_id: &str, - expected_sequence: u64, - events: &[PersistenceEnvelope], - ) -> Result { - Ok(expected_sequence + events.len() as u64) - } - - async fn append_batch( - &self, - appends: &[PersistenceAppend], - ) -> Result, PersistenceError> { - Ok(appends - .iter() - .map(|append| PersistenceAppendResult { - persistence_id: append.persistence_id.clone(), - sequence_nr: append.expected_sequence + append.events.len() as u64, - }) - .collect()) - } - - async fn read_events( - &self, - _persistence_id: &str, - _from_sequence: u64, - ) -> Result, PersistenceError> { - Ok(Vec::new()) - } - - async fn save_snapshot( - &self, - _persistence_id: &str, - _sequence_nr: u64, - _snapshot: &[u8], - ) -> Result<(), PersistenceError> { - self.saves.fetch_add(1, Ordering::SeqCst); - Ok(()) - } - - async fn load_snapshot( - &self, - _persistence_id: &str, - ) -> Result)>, PersistenceError> { - Ok(None) - } - - async fn list_entity_ids( - &self, - _tenant: &str, - ) -> Result, PersistenceError> { - Ok(Vec::new()) - } - - async fn list_entity_ids_by_type( - &self, - _tenant: &str, - _entity_type: &str, - ) -> Result, PersistenceError> { - Ok(Vec::new()) - } - } - - #[test] - fn enqueue_coalesces_newer_snapshot_for_same_stream() { - let queue = SnapshotWriteQueue::new_for_test( - BoxedEventStore::new(RecordingEventStore::default()), - 10, - 10, - ); - - assert_eq!( - queue.enqueue("tenant:Session:s-1".to_string(), 1, vec![1]), - SnapshotEnqueueOutcome::Enqueued - ); - assert_eq!( - queue.enqueue("tenant:Session:s-1".to_string(), 2, vec![2]), - SnapshotEnqueueOutcome::Coalesced - ); - - let writes = queue.take_batch(); - assert_eq!(writes.len(), 1); - assert_eq!(writes[0].sequence_nr, 2); - assert_eq!(writes[0].snapshot, vec![2]); - } - - #[test] - fn enqueue_skips_stale_snapshot_before_store_access() { - let queue = SnapshotWriteQueue::new_for_test( - BoxedEventStore::new(RecordingEventStore::default()), - 10, - 10, - ); - - assert_eq!( - queue.enqueue("tenant:Session:s-1".to_string(), 3, vec![3]), - SnapshotEnqueueOutcome::Enqueued - ); - assert_eq!( - queue.enqueue("tenant:Session:s-1".to_string(), 2, vec![2]), - SnapshotEnqueueOutcome::StaleSkipped - ); - - let writes = queue.take_batch(); - assert_eq!(writes.len(), 1); - assert_eq!(writes[0].sequence_nr, 3); - } - - #[test] - fn enqueue_rejects_new_stream_when_capacity_is_exhausted() { - let queue = SnapshotWriteQueue::new_for_test( - BoxedEventStore::new(RecordingEventStore::default()), - 1, - 10, - ); - - assert_eq!( - queue.enqueue("tenant:Session:s-1".to_string(), 1, vec![1]), - SnapshotEnqueueOutcome::Enqueued - ); - assert_eq!( - queue.enqueue("tenant:Session:s-2".to_string(), 1, vec![1]), - SnapshotEnqueueOutcome::Full - ); - } -} +#[path = "snapshot_queue_test.rs"] +mod tests; diff --git a/crates/temper-server/src/entity_actor/snapshot_queue_test.rs b/crates/temper-server/src/entity_actor/snapshot_queue_test.rs new file mode 100644 index 000000000..7f631a828 --- /dev/null +++ b/crates/temper-server/src/entity_actor/snapshot_queue_test.rs @@ -0,0 +1,632 @@ +use super::*; +use std::sync::atomic::AtomicU8; +use std::sync::atomic::{AtomicUsize, Ordering}; +use temper_runtime::persistence::{ + EventStore, PersistenceAppend, PersistenceAppendResult, PersistenceEnvelope, PersistenceError, + SnapshotSourceFence, encode_activated_key_contract, +}; + +#[derive(Default)] +struct RecordingEventStore { + saves: AtomicUsize, +} + +struct ErroringSnapshotStore { + failure: AtomicU8, +} + +#[cfg(feature = "sim")] +struct BlockingSnapshotStore { + inner: temper_store_sim::SimEventStore, + saves: AtomicUsize, + first_started: tokio::sync::Notify, + resume_first: tokio::sync::Notify, +} + +#[cfg(feature = "sim")] +impl BlockingSnapshotStore { + fn new() -> Self { + Self { + inner: temper_store_sim::SimEventStore::no_faults(45), + saves: AtomicUsize::new(0), + first_started: tokio::sync::Notify::new(), + resume_first: tokio::sync::Notify::new(), + } + } +} + +impl ErroringSnapshotStore { + fn new(failure: u8) -> Self { + Self { + failure: AtomicU8::new(failure), + } + } +} + +impl EventStore for RecordingEventStore { + async fn append( + &self, + _persistence_id: &str, + expected_sequence: u64, + events: &[PersistenceEnvelope], + ) -> Result { + Ok(expected_sequence + events.len() as u64) + } + + async fn append_batch( + &self, + appends: &[PersistenceAppend], + ) -> Result, PersistenceError> { + Ok(appends + .iter() + .map(|append| PersistenceAppendResult { + persistence_id: append.persistence_id.clone(), + sequence_nr: append.expected_sequence + append.events.len() as u64, + batch_already_applied: false, + }) + .collect()) + } + + async fn read_events( + &self, + _persistence_id: &str, + _from_sequence: u64, + ) -> Result, PersistenceError> { + Ok(Vec::new()) + } + + async fn read_events_page( + &self, + _persistence_id: &str, + _from_sequence: u64, + _through_sequence: u64, + _limit: usize, + ) -> Result, PersistenceError> { + Ok(Vec::new()) + } + + async fn save_snapshot( + &self, + _persistence_id: &str, + _sequence_nr: u64, + _snapshot: &[u8], + ) -> Result<(), PersistenceError> { + self.saves.fetch_add(1, Ordering::SeqCst); + Ok(()) + } + + async fn load_snapshot( + &self, + _persistence_id: &str, + ) -> Result)>, PersistenceError> { + Ok(None) + } + + async fn list_entity_ids( + &self, + _tenant: &str, + ) -> Result, PersistenceError> { + Ok(Vec::new()) + } + + async fn list_entity_ids_by_type( + &self, + _tenant: &str, + _entity_type: &str, + ) -> Result, PersistenceError> { + Ok(Vec::new()) + } +} + +impl EventStore for ErroringSnapshotStore { + async fn append( + &self, + _persistence_id: &str, + expected_sequence: u64, + events: &[PersistenceEnvelope], + ) -> Result { + Ok(expected_sequence + events.len() as u64) + } + + async fn append_batch( + &self, + appends: &[PersistenceAppend], + ) -> Result, PersistenceError> { + Ok(appends + .iter() + .map(|append| PersistenceAppendResult { + persistence_id: append.persistence_id.clone(), + sequence_nr: append.expected_sequence + append.events.len() as u64, + batch_already_applied: false, + }) + .collect()) + } + + async fn read_events( + &self, + _persistence_id: &str, + _from_sequence: u64, + ) -> Result, PersistenceError> { + Ok(Vec::new()) + } + + async fn read_events_page( + &self, + _persistence_id: &str, + _from_sequence: u64, + _through_sequence: u64, + _limit: usize, + ) -> Result, PersistenceError> { + Ok(Vec::new()) + } + + async fn save_snapshot( + &self, + _persistence_id: &str, + _sequence_nr: u64, + _snapshot: &[u8], + ) -> Result<(), PersistenceError> { + Ok(()) + } + + async fn save_snapshot_if_source( + &self, + _persistence_id: &str, + _sequence_nr: u64, + _snapshot: &[u8], + _source: &SnapshotSourceFence, + _key_contract: Option<&str>, + ) -> Result<(), PersistenceError> { + match self.failure.load(Ordering::SeqCst) { + 1 => Err(PersistenceError::SnapshotGenerationChanged), + 2 => Err(PersistenceError::KeyContractNotActive { + activated_signature: "active".to_string(), + attempted_signature: "stale".to_string(), + }), + 3 => Err(PersistenceError::KeyContractActivationNotReady { + activated_epoch: 2, + activated_signature: "active".to_string(), + }), + _ => Ok(()), + } + } + + async fn load_snapshot( + &self, + _persistence_id: &str, + ) -> Result)>, PersistenceError> { + Ok(None) + } + + async fn list_entity_ids( + &self, + _tenant: &str, + ) -> Result, PersistenceError> { + Ok(Vec::new()) + } + + async fn list_entity_ids_by_type( + &self, + _tenant: &str, + _entity_type: &str, + ) -> Result, PersistenceError> { + Ok(Vec::new()) + } +} + +#[cfg(feature = "sim")] +impl EventStore for BlockingSnapshotStore { + async fn append( + &self, + persistence_id: &str, + expected_sequence: u64, + events: &[PersistenceEnvelope], + ) -> Result { + self.inner + .append(persistence_id, expected_sequence, events) + .await + } + + async fn append_batch( + &self, + appends: &[PersistenceAppend], + ) -> Result, PersistenceError> { + self.inner.append_batch(appends).await + } + + async fn read_events( + &self, + persistence_id: &str, + from_sequence: u64, + ) -> Result, PersistenceError> { + self.inner.read_events(persistence_id, from_sequence).await + } + + async fn read_events_page( + &self, + persistence_id: &str, + from_sequence: u64, + through_sequence: u64, + limit: usize, + ) -> Result, PersistenceError> { + self.inner + .read_events_page(persistence_id, from_sequence, through_sequence, limit) + .await + } + + async fn save_snapshot( + &self, + persistence_id: &str, + sequence_nr: u64, + snapshot: &[u8], + ) -> Result<(), PersistenceError> { + self.inner + .save_snapshot(persistence_id, sequence_nr, snapshot) + .await + } + + async fn save_snapshot_if_source( + &self, + persistence_id: &str, + sequence_nr: u64, + snapshot: &[u8], + source: &SnapshotSourceFence, + key_contract: Option<&str>, + ) -> Result<(), PersistenceError> { + if self.saves.fetch_add(1, Ordering::SeqCst) == 0 { + self.first_started.notify_one(); + self.resume_first.notified().await; + } + self.inner + .save_snapshot_if_source(persistence_id, sequence_nr, snapshot, source, key_contract) + .await + } + + async fn load_snapshot( + &self, + persistence_id: &str, + ) -> Result)>, PersistenceError> { + self.inner.load_snapshot(persistence_id).await + } + + async fn list_entity_ids( + &self, + tenant: &str, + ) -> Result, PersistenceError> { + self.inner.list_entity_ids(tenant).await + } + + async fn list_entity_ids_by_type( + &self, + tenant: &str, + entity_type: &str, + ) -> Result, PersistenceError> { + self.inner + .list_entity_ids_by_type(tenant, entity_type) + .await + } +} + +fn activated_contract(epoch: u64) -> String { + encode_activated_key_contract("v3|path:WorkspaceId,Path", epoch) +} + +#[test] +fn enqueue_coalesces_newer_snapshot_for_same_stream() { + let queue = SnapshotWriteQueue::new_for_test( + BoxedEventStore::new(RecordingEventStore::default()), + 10, + 10, + ); + + assert_eq!( + queue.enqueue( + "tenant:Session:s-1".to_string(), + 1, + vec![1], + SnapshotSourceFence::Unchecked, + None, + ), + SnapshotEnqueueOutcome::Enqueued + ); + assert_eq!( + queue.enqueue( + "tenant:Session:s-1".to_string(), + 2, + vec![2], + SnapshotSourceFence::Unchecked, + None, + ), + SnapshotEnqueueOutcome::Coalesced + ); + + let writes = queue.take_batch(); + assert_eq!(writes.len(), 1); + assert_eq!(writes[0].sequence_nr, 2); + assert_eq!(writes[0].snapshot, vec![2]); +} + +#[test] +fn enqueue_skips_stale_snapshot_before_store_access() { + let queue = SnapshotWriteQueue::new_for_test( + BoxedEventStore::new(RecordingEventStore::default()), + 10, + 10, + ); + + assert_eq!( + queue.enqueue( + "tenant:Session:s-1".to_string(), + 3, + vec![3], + SnapshotSourceFence::Unchecked, + None, + ), + SnapshotEnqueueOutcome::Enqueued + ); + assert_eq!( + queue.enqueue( + "tenant:Session:s-1".to_string(), + 2, + vec![2], + SnapshotSourceFence::Unchecked, + None, + ), + SnapshotEnqueueOutcome::StaleSkipped + ); + + let writes = queue.take_batch(); + assert_eq!(writes.len(), 1); + assert_eq!(writes[0].sequence_nr, 3); +} + +#[test] +fn enqueue_skips_snapshot_below_exact_source_sequence() { + let queue = SnapshotWriteQueue::new_for_test( + BoxedEventStore::new(RecordingEventStore::default()), + 10, + 10, + ); + + assert_eq!( + queue.enqueue( + "tenant:Session:snapshot-ahead".to_string(), + 2, + vec![2], + SnapshotSourceFence::Exact { + sequence_nr: 5, + state: vec![5], + }, + None, + ), + SnapshotEnqueueOutcome::StaleSkipped + ); + assert_eq!( + queue.pending_sequence("tenant:Session:snapshot-ahead"), + None, + "the queue must not report an older monotonic no-op as pending" + ); + assert!(queue.take_batch().is_empty()); +} + +#[test] +fn enqueue_rejects_new_stream_when_capacity_is_exhausted() { + let queue = SnapshotWriteQueue::new_for_test( + BoxedEventStore::new(RecordingEventStore::default()), + 1, + 10, + ); + + assert_eq!( + queue.enqueue( + "tenant:Session:s-1".to_string(), + 1, + vec![1], + SnapshotSourceFence::Unchecked, + None, + ), + SnapshotEnqueueOutcome::Enqueued + ); + assert_eq!( + queue.enqueue( + "tenant:Session:s-2".to_string(), + 1, + vec![1], + SnapshotSourceFence::Unchecked, + None, + ), + SnapshotEnqueueOutcome::Full + ); +} + +#[test] +fn higher_epoch_replaces_equal_sequence_pending_snapshot() { + let queue = SnapshotWriteQueue::new_for_test( + BoxedEventStore::new(RecordingEventStore::default()), + 10, + 10, + ); + let persistence_id = "tenant:Doc:epoch-replacement"; + assert_eq!( + queue.enqueue( + persistence_id.to_string(), + 7, + vec![1], + SnapshotSourceFence::Absent, + Some(activated_contract(1)), + ), + SnapshotEnqueueOutcome::Enqueued + ); + assert_eq!( + queue.enqueue( + persistence_id.to_string(), + 7, + vec![2], + SnapshotSourceFence::Absent, + Some(activated_contract(2)), + ), + SnapshotEnqueueOutcome::Coalesced + ); + let writes = queue.take_batch(); + assert_eq!(writes.len(), 1); + assert_eq!(writes[0].snapshot, vec![2]); + assert_eq!(writes[0].key_contract, Some(activated_contract(2))); +} + +#[test] +fn lower_epoch_cannot_replace_or_suppress_higher_epoch_snapshot() { + let queue = SnapshotWriteQueue::new_for_test( + BoxedEventStore::new(RecordingEventStore::default()), + 10, + 10, + ); + let persistence_id = "tenant:Doc:epoch-order"; + assert_eq!( + queue.enqueue( + persistence_id.to_string(), + 4, + vec![2], + SnapshotSourceFence::Absent, + Some(activated_contract(2)), + ), + SnapshotEnqueueOutcome::Enqueued + ); + assert_eq!( + queue.enqueue( + persistence_id.to_string(), + 99, + vec![1], + SnapshotSourceFence::Absent, + Some(activated_contract(1)), + ), + SnapshotEnqueueOutcome::StaleSkipped + ); + let writes = queue.take_batch(); + assert_eq!(writes.len(), 1); + assert_eq!(writes[0].key_contract, Some(activated_contract(2))); +} + +#[test] +fn old_epoch_pending_and_applied_boundaries_do_not_throttle_new_epoch() { + let queue = SnapshotWriteQueue::new_for_test( + BoxedEventStore::new(RecordingEventStore::default()), + 10, + 10, + ); + let persistence_id = "tenant:Doc:epoch-boundary"; + let old_contract = activated_contract(1); + let new_contract = activated_contract(2); + queue.enqueue( + persistence_id.to_string(), + 100, + vec![1], + SnapshotSourceFence::Absent, + Some(old_contract.clone()), + ); + queue.record_applied_sequence(persistence_id, 100, Some(&old_contract)); + + assert_eq!( + queue.pending_sequence_for_contract(persistence_id, Some(&new_contract)), + None + ); + assert_eq!( + queue.applied_sequence_for_contract(persistence_id, Some(&new_contract)), + None + ); + assert_eq!( + queue.pending_sequence_for_contract(persistence_id, Some(&old_contract)), + Some(100) + ); + assert_eq!( + queue.applied_sequence_for_contract(persistence_id, Some(&old_contract)), + Some(100) + ); +} + +#[cfg(feature = "sim")] +#[tokio::test] +async fn in_flight_snapshot_rebases_the_already_pending_next_write() { + let store = Arc::new(BlockingSnapshotStore::new()); + let queue = Arc::new(SnapshotWriteQueue::new_for_test( + BoxedEventStore::from_arc(store.clone()), + 10, + 10, + )); + let persistence_id = "tenant:Doc:in-flight-source-chain"; + assert_eq!( + queue.enqueue( + persistence_id.to_string(), + 1, + vec![1], + SnapshotSourceFence::Absent, + None, + ), + SnapshotEnqueueOutcome::Enqueued + ); + let first = queue.take_batch().pop().expect("first queued write"); + let apply_queue = queue.clone(); + let first_apply = tokio::spawn(async move { apply_queue.apply(first).await }); + store.first_started.notified().await; + + assert_eq!( + queue.enqueue( + persistence_id.to_string(), + 2, + vec![2], + SnapshotSourceFence::Absent, + None, + ), + SnapshotEnqueueOutcome::Enqueued, + "the second write must be pending before the first one commits" + ); + store.resume_first.notify_one(); + first_apply.await.expect("first apply task"); + + let second = queue.take_batch().pop().expect("second queued write"); + queue.apply(second).await; + assert_eq!(queue.applied_sequence(persistence_id), Some(2)); + assert_eq!( + store + .load_snapshot(persistence_id) + .await + .expect("load final snapshot"), + Some((2, vec![2])) + ); +} + +#[tokio::test] +async fn terminal_contract_errors_drop_but_not_ready_requeues() { + for terminal_failure in [1, 2] { + let queue = SnapshotWriteQueue::new_for_test( + BoxedEventStore::new(ErroringSnapshotStore::new(terminal_failure)), + 10, + 10, + ); + queue.enqueue( + format!("tenant:Doc:terminal-{terminal_failure}"), + 1, + vec![1], + SnapshotSourceFence::Absent, + Some(activated_contract(1)), + ); + let write = queue.take_batch().pop().expect("queued terminal write"); + queue.apply(write).await; + assert!(queue.take_batch().is_empty()); + } + + let queue = SnapshotWriteQueue::new_for_test( + BoxedEventStore::new(ErroringSnapshotStore::new(3)), + 10, + 10, + ); + queue.enqueue( + "tenant:Doc:not-ready".to_string(), + 1, + vec![1], + SnapshotSourceFence::Absent, + Some(activated_contract(2)), + ); + let write = queue.take_batch().pop().expect("queued not-ready write"); + queue.apply(write).await; + assert_eq!(queue.pending_sequence("tenant:Doc:not-ready"), Some(1)); +} diff --git a/crates/temper-server/src/entity_actor/types.rs b/crates/temper-server/src/entity_actor/types.rs index a3e23d142..8a06a1389 100644 --- a/crates/temper-server/src/entity_actor/types.rs +++ b/crates/temper-server/src/entity_actor/types.rs @@ -5,6 +5,7 @@ use std::sync::OnceLock; use serde::{Deserialize, Serialize}; use temper_runtime::actor::Message; +use temper_runtime::persistence::SnapshotSourceFence; // TigerStyle: Fixed resource budgets. No unbounded growth. // These are hard limits, not suggestions. Violations are assertion failures. @@ -18,7 +19,8 @@ pub const RECENT_EVENTS_BUDGET_DEFAULT: usize = 50; /// Maximum items an entity can hold. pub const MAX_ITEMS_PER_ENTITY: usize = 1_000; /// Maximum durable idempotency keys retained per entity. -pub const MAX_DURABLE_IDEMPOTENCY_KEYS_PER_ENTITY: usize = 1_000; +pub const MAX_DURABLE_IDEMPOTENCY_KEYS_PER_ENTITY: usize = + temper_runtime::persistence::STATE_MATERIALIZATION_IDEMPOTENCY_KEY_BUDGET; /// Number of recent events retained in memory per entity. /// @@ -51,12 +53,20 @@ pub enum EntityMsg { }, /// Get the current entity state. GetState, + /// Get state plus the exact snapshot generation used to hydrate it. + /// + /// This internal response lets passivation condition its snapshot write on + /// the same source instead of overwriting a concurrent same-sequence rewrite. + GetPassivationSnapshot, /// Get a specific field value. GetField { field: String }, /// Update entity fields (PATCH: merge, PUT: replace). UpdateFields { fields: serde_json::Value, replace: bool, + /// Request-stable token used to deduplicate dispatch retries after a + /// durable append succeeds but the actor reply is lost or delayed. + idempotency_key: String, }, /// Delete this entity. Delete, @@ -64,6 +74,14 @@ pub enum EntityMsg { impl Message for EntityMsg {} +/// Actor state and exact snapshot source captured in one mailbox turn. +#[derive(Debug, Clone)] +pub(crate) struct EntityPassivationSnapshot { + pub(crate) state: EntityState, + pub(crate) snapshot_source: SnapshotSourceFence, + pub(crate) key_contract: String, +} + /// The entity's runtime state. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct EntityState { @@ -130,12 +148,22 @@ impl EntityState { } } + /// Charge one durable non-domain envelope against the unsnapshotted journal budget. + pub(crate) fn record_internal_envelope(&mut self) { + self.events_since_snapshot = self.events_since_snapshot.saturating_add(1); + } + pub fn has_processed_idempotency_key(&self, key: &str) -> bool { self.processed_idempotency_keys.contains_key(key) } fn record_processed_idempotency_key(&mut self, key: &str) { let sequence = self.sequence_nr.max(self.total_event_count as u64); + self.record_durable_idempotency_key(key, sequence); + } + + /// Record a durable non-domain operation at its exact journal sequence. + pub(crate) fn record_durable_idempotency_key(&mut self, key: &str, sequence: u64) { self.processed_idempotency_keys .insert(key.to_string(), sequence); diff --git a/crates/temper-server/src/idempotency.rs b/crates/temper-server/src/idempotency.rs index d3479f818..404c9c6ef 100644 --- a/crates/temper-server/src/idempotency.rs +++ b/crates/temper-server/src/idempotency.rs @@ -20,12 +20,73 @@ pub const IDEMPOTENCY_TTL_SECS: i64 = 3600; /// A cached idempotent response. struct IdempotencyEntry { /// The cached response to return on duplicate requests. - response: EntityResponse, + response: Option, /// When this entry was created (for TTL eviction). created_at: chrono::DateTime, /// Whether dispatcher-side post-dispatch effects have completed for this /// cached response. effects_applied: bool, + /// Canonical action + params + principal proof for safe post-action replay. + bound_action_fingerprint: Option, + /// Original action parameters paired with `bound_action_fingerprint`. + bound_action_params: Option, + /// Whether the bound-action post-action hook completed successfully. + bound_action_hook_completed: bool, + /// Whether one request currently owns execution of the post-action hook. + bound_action_hook_in_flight: bool, + /// Exact hook output merged into the successful protocol response. + bound_action_hook_output: Option, + /// Keep an exact post-action replay proof while an outcome-ambiguous + /// publication debt is armed. The sticky tenant gate can outlive the normal + /// cache TTL, and evicting its only memory-mode recovery proof would make + /// the gate impossible to discharge. + publication_replay_pinned: bool, +} + +/// Result of looking up an effects-applied bound action for post-action replay. +#[derive(Debug, Clone)] +pub enum BoundActionReplayLookup { + /// No live, effects-applied bound-action proof exists for this key. + Miss, + /// The retry exactly matches the originally authorized action request. + Match { + /// Cached entity response produced by the governed action. + response: Box, + /// Original parameters supplied to the post-action hook. + params: serde_json::Value, + /// Whether the post-action hook has already completed. + hook_completed: bool, + /// Cached post-action output from the completed hook. + hook_output: Option, + }, + /// The exact replay proof exists, but another request owns its hook. + Pending, + /// The idempotency key exists but action, parameters, or principal differ. + Conflict, +} + +/// Atomic result of reserving a raw bound-action idempotency key. +#[derive(Debug, Clone)] +pub enum BoundActionClaim { + /// This request owns the new reservation and may dispatch the action. + Claimed, + /// The exact same request already owns an unfinished reservation. + Pending, + /// The exact request has a completed actor response whose hook may replay. + Match { + /// Cached entity response produced by the governed action. + response: Box, + /// Original parameters supplied to the post-action hook. + params: serde_json::Value, + /// Whether the post-action hook has already completed. + hook_completed: bool, + /// Cached post-action output from the completed hook. + hook_output: Option, + }, + /// The raw key is already owned by another request or by an unproved legacy entry. + Conflict, + /// Every actor-local cache slot is protected by unfinished work. + AtCapacity, } /// Per-entity-actor idempotency cache. @@ -37,6 +98,78 @@ pub struct IdempotencyCache { entries: RwLock>>, } +/// Cancellation guard for one newly claimed raw bound-action key. +#[must_use = "dropping the guard releases the raw bound-action reservation"] +pub(crate) struct BoundActionReservationGuard<'a> { + cache: &'a IdempotencyCache, + actor_key: String, + idem_key: String, + request_fingerprint: String, + armed: bool, +} + +impl BoundActionReservationGuard<'_> { + /// Disarm after the raw reservation has been atomically replaced by its + /// completed actor response. + pub(crate) fn disarm(mut self) { + self.armed = false; + } +} + +impl Drop for BoundActionReservationGuard<'_> { + fn drop(&mut self) { + if self.armed { + self.cache.abandon_bound_action_claim( + &self.actor_key, + &self.idem_key, + &self.request_fingerprint, + ); + } + } +} + +/// Cancellation guard for one owned post-action hook replay. +#[must_use = "dropping the guard releases the post-action hook claim"] +pub(crate) struct BoundActionHookGuard<'a, F> +where + F: Fn() -> bool, +{ + cache: &'a IdempotencyCache, + actor_key: String, + idem_key: String, + request_fingerprint: String, + publication_gated: F, + armed: bool, +} + +impl BoundActionHookGuard<'_, F> +where + F: Fn() -> bool, +{ + /// Disarm after durable receipt persistence and cache completion succeed. + pub(crate) fn disarm(mut self) { + self.armed = false; + } +} + +impl Drop for BoundActionHookGuard<'_, F> +where + F: Fn() -> bool, +{ + fn drop(&mut self) { + if !self.armed { + return; + } + let publication_gated = (self.publication_gated)(); + self.cache.release_bound_action_hook( + &self.actor_key, + &self.idem_key, + &self.request_fingerprint, + publication_gated, + ); + } +} + impl IdempotencyCache { /// Create a new empty idempotency cache. pub fn new() -> Self { @@ -45,6 +178,46 @@ impl IdempotencyCache { } } + /// Arm cancellation cleanup for a raw bound-action reservation owned by + /// the current request. + pub(crate) fn guard_bound_action_reservation( + &self, + actor_key: &str, + idem_key: &str, + request_fingerprint: &str, + ) -> BoundActionReservationGuard<'_> { + BoundActionReservationGuard { + cache: self, + actor_key: actor_key.to_string(), + idem_key: idem_key.to_string(), + request_fingerprint: request_fingerprint.to_string(), + armed: true, + } + } + + /// Arm cancellation cleanup for an owned hook replay. The supplied gate + /// check is evaluated at drop time so a hook that arms publication debt + /// before cancellation retains its exact recovery proof. + pub(crate) fn guard_bound_action_hook( + &self, + actor_key: &str, + idem_key: &str, + request_fingerprint: &str, + publication_gated: F, + ) -> BoundActionHookGuard<'_, F> + where + F: Fn() -> bool, + { + BoundActionHookGuard { + cache: self, + actor_key: actor_key.to_string(), + idem_key: idem_key.to_string(), + request_fingerprint: request_fingerprint.to_string(), + publication_gated, + armed: true, + } + } + /// Look up a cached response. Returns `None` if not found or expired. pub fn get(&self, actor_key: &str, idem_key: &str) -> Option { let now = sim_now(); @@ -61,7 +234,7 @@ impl IdempotencyCache { return None; } - Some(entry.response.clone()) + entry.response.clone() } /// Look up a cached response only after post-dispatch effects have run. @@ -75,7 +248,10 @@ impl IdempotencyCache { idem_key: &str, ) -> Option { let now = sim_now(); - let entries = self.entries.read().unwrap(); // ci-ok: infallible lock + let entries = self + .entries + .read() + .expect("idempotency cache lock poisoned"); let actor_entries = entries.get(actor_key)?; let entry = actor_entries.get(idem_key)?; @@ -84,19 +260,349 @@ impl IdempotencyCache { return None; } - Some(entry.response.clone()) + entry.response.clone() } /// Cache a response for a given actor and idempotency key. /// /// If the per-actor budget is exceeded, the oldest entry is evicted. pub fn put(&self, actor_key: &str, idem_key: &str, response: EntityResponse) { - self.put_with_effects_applied(actor_key, idem_key, response, false); + let _ = self.put_with_effects_applied(actor_key, idem_key, response, false, None); } /// Cache a response whose post-dispatch effects are known to be complete. pub fn put_effects_applied(&self, actor_key: &str, idem_key: &str, response: EntityResponse) { - self.put_with_effects_applied(actor_key, idem_key, response, true); + let _ = self.put_with_effects_applied(actor_key, idem_key, response, true, None); + } + + /// Cache an effects-applied bound action together with the exact request + /// proof required to replay its post-action hook safely. + pub fn put_bound_action_effects_applied( + &self, + actor_key: &str, + idem_key: &str, + response: EntityResponse, + request_fingerprint: String, + params: serde_json::Value, + ) -> bool { + self.put_with_effects_applied( + actor_key, + idem_key, + response, + true, + Some((request_fingerprint, params)), + ) + } + + /// Look up a post-action replay and reject reuse of an idempotency key by a + /// different action request or principal. + pub fn lookup_bound_action_replay( + &self, + actor_key: &str, + idem_key: &str, + request_fingerprint: &str, + ) -> BoundActionReplayLookup { + let now = sim_now(); + let mut entries = self + .entries + .write() + .expect("idempotency cache lock poisoned"); + let Some(entry) = entries + .get_mut(actor_key) + .and_then(|actor_entries| actor_entries.get_mut(idem_key)) + else { + return BoundActionReplayLookup::Miss; + }; + let age = now.signed_duration_since(entry.created_at); + if age.num_seconds() > IDEMPOTENCY_TTL_SECS + && !entry.publication_replay_pinned + && !entry.bound_action_hook_in_flight + { + return BoundActionReplayLookup::Miss; + } + match ( + entry.bound_action_fingerprint.as_deref(), + entry.bound_action_params.as_ref(), + ) { + (Some(stored), Some(params)) if stored == request_fingerprint => { + match entry.response.as_ref() { + Some(response) + if entry.effects_applied && entry.bound_action_hook_completed => + { + BoundActionReplayLookup::Match { + response: Box::new(response.clone()), + params: params.clone(), + hook_completed: true, + hook_output: entry.bound_action_hook_output.clone(), + } + } + Some(_) if entry.effects_applied && entry.bound_action_hook_in_flight => { + BoundActionReplayLookup::Pending + } + Some(response) if entry.effects_applied => { + entry.bound_action_hook_in_flight = true; + BoundActionReplayLookup::Match { + response: Box::new(response.clone()), + params: params.clone(), + hook_completed: false, + hook_output: None, + } + } + _ => BoundActionReplayLookup::Miss, + } + } + (Some(_), Some(_)) => BoundActionReplayLookup::Conflict, + _ => BoundActionReplayLookup::Conflict, + } + } + + /// Atomically reserve a raw idempotency key for one exact bound-action + /// request. This closes the interval between protocol lookup and actor + /// completion in which another action could otherwise reuse the key. + pub fn claim_bound_action( + &self, + actor_key: &str, + idem_key: &str, + request_fingerprint: &str, + params: &serde_json::Value, + ) -> BoundActionClaim { + let now = sim_now(); + let mut entries = self + .entries + .write() + .expect("idempotency cache lock poisoned"); + let actor_entries = entries.entry(actor_key.to_string()).or_default(); + actor_entries.retain(|_, entry| { + Self::entry_is_protected(entry) + || now.signed_duration_since(entry.created_at).num_seconds() <= IDEMPOTENCY_TTL_SECS + }); + + if let Some(entry) = actor_entries.get_mut(idem_key) { + return match ( + entry.bound_action_fingerprint.as_deref(), + entry.bound_action_params.as_ref(), + ) { + (Some(stored), Some(stored_params)) if stored == request_fingerprint => { + match entry.response.as_ref() { + Some(response) + if entry.effects_applied && entry.bound_action_hook_completed => + { + BoundActionClaim::Match { + response: Box::new(response.clone()), + params: stored_params.clone(), + hook_completed: true, + hook_output: entry.bound_action_hook_output.clone(), + } + } + Some(_) if entry.effects_applied && entry.bound_action_hook_in_flight => { + BoundActionClaim::Pending + } + Some(response) if entry.effects_applied => { + entry.bound_action_hook_in_flight = true; + BoundActionClaim::Match { + response: Box::new(response.clone()), + params: stored_params.clone(), + hook_completed: false, + hook_output: None, + } + } + _ => BoundActionClaim::Pending, + } + } + _ => BoundActionClaim::Conflict, + }; + } + + if !Self::make_room_for_new_entry(actor_entries) { + return BoundActionClaim::AtCapacity; + } + actor_entries.insert( + idem_key.to_string(), + IdempotencyEntry { + response: None, + created_at: now, + effects_applied: false, + bound_action_fingerprint: Some(request_fingerprint.to_string()), + bound_action_params: Some(params.clone()), + bound_action_hook_completed: false, + bound_action_hook_in_flight: false, + bound_action_hook_output: None, + publication_replay_pinned: false, + }, + ); + BoundActionClaim::Claimed + } + + /// Mark one exact bound-action hook successful and retain its response + /// output so later protocol retries do not repeat external work. + pub fn complete_bound_action_hook( + &self, + actor_key: &str, + idem_key: &str, + request_fingerprint: &str, + hook_output: Option, + ) -> bool { + let mut entries = self + .entries + .write() + .expect("idempotency cache lock poisoned"); + let Some(entry) = entries + .get_mut(actor_key) + .and_then(|actor_entries| actor_entries.get_mut(idem_key)) + else { + return false; + }; + if !entry.effects_applied + || entry.response.is_none() + || entry.bound_action_fingerprint.as_deref() != Some(request_fingerprint) + { + return false; + } + entry.bound_action_hook_completed = true; + entry.bound_action_hook_in_flight = false; + entry.bound_action_hook_output = hook_output; + entry.publication_replay_pinned = false; + entry.created_at = sim_now(); + true + } + + /// Release one failed hook claim so an exact retry can own it. + pub fn fail_bound_action_hook( + &self, + actor_key: &str, + idem_key: &str, + request_fingerprint: &str, + ) { + self.release_bound_action_hook(actor_key, idem_key, request_fingerprint, false); + } + + fn release_bound_action_hook( + &self, + actor_key: &str, + idem_key: &str, + request_fingerprint: &str, + pin_publication_replay: bool, + ) { + let mut entries = self + .entries + .write() + .expect("idempotency cache lock poisoned"); + if let Some(entry) = entries + .get_mut(actor_key) + .and_then(|actor_entries| actor_entries.get_mut(idem_key)) + && entry.effects_applied + && !entry.bound_action_hook_completed + && entry.bound_action_hook_in_flight + && entry.bound_action_fingerprint.as_deref() == Some(request_fingerprint) + { + entry.bound_action_hook_in_flight = false; + entry.publication_replay_pinned |= pin_publication_replay; + entry.created_at = sim_now(); + } + } + + /// Pin the exact completed replay proof responsible for an armed + /// publication debt until its retry succeeds. + pub fn pin_bound_action_replay( + &self, + actor_key: &str, + idem_key: &str, + request_fingerprint: &str, + ) -> bool { + let mut entries = self + .entries + .write() + .expect("idempotency cache lock poisoned"); + let Some(entry) = entries + .get_mut(actor_key) + .and_then(|actor_entries| actor_entries.get_mut(idem_key)) + else { + return false; + }; + if entry.effects_applied + && entry.response.is_some() + && entry.bound_action_fingerprint.as_deref() == Some(request_fingerprint) + { + entry.publication_replay_pinned = true; + return true; + } + false + } + + /// Return a completed publication replay to ordinary bounded-cache policy + /// after its sticky debt has been discharged. Incomplete or in-flight hook + /// claims retain the pin because cancellation can still require recovery. + pub fn unpin_bound_action_replay( + &self, + actor_key: &str, + idem_key: &str, + request_fingerprint: &str, + ) { + let mut entries = self + .entries + .write() + .expect("idempotency cache lock poisoned"); + if let Some(entry) = entries + .get_mut(actor_key) + .and_then(|actor_entries| actor_entries.get_mut(idem_key)) + && entry.bound_action_fingerprint.as_deref() == Some(request_fingerprint) + && entry.bound_action_hook_completed + && !entry.bound_action_hook_in_flight + { + entry.publication_replay_pinned = false; + entry.created_at = sim_now(); + } + } + + /// Release an unfinished reservation after dispatch fails without a + /// durable action response. A newer owner or completed response is retained. + pub fn abandon_bound_action_claim( + &self, + actor_key: &str, + idem_key: &str, + request_fingerprint: &str, + ) { + let mut entries = self + .entries + .write() + .expect("idempotency cache lock poisoned"); + let Some(actor_entries) = entries.get_mut(actor_key) else { + return; + }; + let remove = actor_entries.get(idem_key).is_some_and(|entry| { + entry.response.is_none() + && entry.bound_action_fingerprint.as_deref() == Some(request_fingerprint) + }); + if remove { + actor_entries.remove(idem_key); + } + } + + #[cfg(test)] + pub(crate) fn clear_actor_for_test(&self, actor_key: &str) { + self.entries.write().unwrap().remove(actor_key); // ci-ok: test-only lock + } + + /// Whether this key names a live, effects-applied bound action. Admission + /// uses this only to reach the handler, which still verifies the fingerprint. + pub fn has_bound_action_replay(&self, actor_key: &str, idem_key: &str) -> bool { + let now = sim_now(); + let entries = self + .entries + .read() + .expect("idempotency cache lock poisoned"); + entries + .get(actor_key) + .and_then(|actor_entries| actor_entries.get(idem_key)) + .is_some_and(|entry| { + (entry.publication_replay_pinned + || now.signed_duration_since(entry.created_at).num_seconds() + <= IDEMPOTENCY_TTL_SECS) + && entry.effects_applied + && entry.response.is_some() + && entry.bound_action_fingerprint.is_some() + && entry.bound_action_params.is_some() + }) } /// Mark a cached response as having completed post-dispatch effects. @@ -128,38 +634,92 @@ impl IdempotencyCache { idem_key: &str, response: EntityResponse, effects_applied: bool, - ) { + bound_action: Option<(String, serde_json::Value)>, + ) -> bool { let now = sim_now(); - let mut entries = self.entries.write().unwrap(); // ci-ok: infallible lock + let mut entries = self + .entries + .write() + .expect("idempotency cache lock poisoned"); let actor_entries = entries.entry(actor_key.to_string()).or_default(); // Evict expired entries first. actor_entries.retain(|_, entry| { - now.signed_duration_since(entry.created_at).num_seconds() <= IDEMPOTENCY_TTL_SECS + Self::entry_is_protected(entry) + || now.signed_duration_since(entry.created_at).num_seconds() <= IDEMPOTENCY_TTL_SECS + }); + + if !actor_entries.contains_key(idem_key) && !Self::make_room_for_new_entry(actor_entries) { + return false; + } + + let inherited_bound_action = actor_entries.get(idem_key).and_then(|entry| { + entry + .bound_action_fingerprint + .clone() + .zip(entry.bound_action_params.clone()) + }); + let inherited_pin = actor_entries + .get(idem_key) + .is_some_and(|entry| entry.publication_replay_pinned); + let inherited_hook = actor_entries.get(idem_key).map(|entry| { + ( + entry.bound_action_hook_completed, + entry.bound_action_hook_in_flight, + entry.bound_action_hook_output.clone(), + ) }); + let resets_bound_action_hook = bound_action.is_some(); + let bound_action = bound_action.or(inherited_bound_action); + let (bound_action_hook_completed, bound_action_hook_in_flight, bound_action_hook_output) = + if resets_bound_action_hook { + (false, true, None) + } else { + inherited_hook.unwrap_or((false, false, None)) + }; + actor_entries.insert( + idem_key.to_string(), + IdempotencyEntry { + response: Some(response), + created_at: now, + effects_applied, + bound_action_fingerprint: bound_action + .as_ref() + .map(|(fingerprint, _)| fingerprint.clone()), + bound_action_params: bound_action.map(|(_, params)| params), + bound_action_hook_completed, + bound_action_hook_in_flight, + bound_action_hook_output, + publication_replay_pinned: inherited_pin, + }, + ); + true + } - // Budget enforcement: evict oldest if at capacity. + fn make_room_for_new_entry(actor_entries: &mut BTreeMap) -> bool { + // Budget enforcement: evict oldest replaceable entries until a new + // admission fits. Active reservations are protected because evicting + // one would allow the same raw key to be claimed concurrently. while actor_entries.len() >= IDEMPOTENCY_BUDGET_PER_ACTOR { - // Find the oldest entry by created_at. if let Some(oldest_key) = actor_entries .iter() + .filter(|(_, entry)| !Self::entry_is_protected(entry)) .min_by_key(|(_, e)| e.created_at) .map(|(k, _)| k.clone()) { actor_entries.remove(&oldest_key); } else { - break; + return false; } } + debug_assert!(actor_entries.len() < IDEMPOTENCY_BUDGET_PER_ACTOR); + true + } - actor_entries.insert( - idem_key.to_string(), - IdempotencyEntry { - response, - created_at: now, - effects_applied, - }, - ); + fn entry_is_protected(entry: &IdempotencyEntry) -> bool { + entry.publication_replay_pinned + || entry.bound_action_hook_in_flight + || (entry.response.is_none() && entry.bound_action_fingerprint.is_some()) } } @@ -170,104 +730,5 @@ impl Default for IdempotencyCache { } #[cfg(test)] -mod tests { - use super::*; - use crate::entity_actor::{EntityResponse, EntityState}; - use std::collections::BTreeMap; - - fn make_response(status: &str) -> EntityResponse { - EntityResponse { - success: true, - state: EntityState { - entity_type: String::new(), - entity_id: String::new(), - status: status.to_string(), - item_count: 0, - counters: BTreeMap::new(), - booleans: BTreeMap::new(), - lists: BTreeMap::new(), - fields: serde_json::json!({}), - events: std::collections::VecDeque::new(), - total_event_count: 0, - events_since_snapshot: 0, - last_snapshot_sequence_nr: 0, - sequence_nr: 0, - processed_idempotency_keys: BTreeMap::new(), - }, - error: None, - custom_effects: vec![], - scheduled_actions: vec![], - spawn_requests: vec![], - spec_governed: true, - } - } - - #[test] - fn put_then_get_returns_cached() { - let cache = IdempotencyCache::new(); - let resp = make_response("Active"); - cache.put("Order:o1", "key-1", resp.clone()); - let cached = cache.get("Order:o1", "key-1"); - assert!(cached.is_some()); - assert_eq!(cached.unwrap().state.status, "Active"); - } - - #[test] - fn pending_effects_do_not_satisfy_protocol_cache_hit() { - let cache = IdempotencyCache::new(); - cache.put("Order:o1", "key-1", make_response("Active")); - - assert!(cache.get("Order:o1", "key-1").is_some()); - assert!( - cache - .get_after_effects_applied("Order:o1", "key-1") - .is_none() - ); - - assert!(cache.mark_effects_applied("Order:o1", "key-1")); - assert!( - cache - .get_after_effects_applied("Order:o1", "key-1") - .is_some() - ); - } - - #[test] - fn put_effects_applied_satisfies_protocol_cache_hit() { - let cache = IdempotencyCache::new(); - cache.put_effects_applied("Order:o1", "key-1", make_response("Active")); - - let cached = cache.get_after_effects_applied("Order:o1", "key-1"); - assert_eq!(cached.unwrap().state.status, "Active"); - } - - #[test] - fn get_missing_returns_none() { - let cache = IdempotencyCache::new(); - assert!(cache.get("Order:o1", "no-such-key").is_none()); - } - - #[test] - fn different_actors_isolated() { - let cache = IdempotencyCache::new(); - cache.put("Order:o1", "key-1", make_response("A")); - cache.put("Order:o2", "key-1", make_response("B")); - assert_eq!(cache.get("Order:o1", "key-1").unwrap().state.status, "A"); - assert_eq!(cache.get("Order:o2", "key-1").unwrap().state.status, "B"); - } - - #[test] - fn budget_evicts_oldest() { - let cache = IdempotencyCache::new(); - // Fill to budget - for i in 0..IDEMPOTENCY_BUDGET_PER_ACTOR { - cache.put("actor", &format!("k-{i}"), make_response("S")); - } - // One more should evict the oldest - cache.put("actor", "k-overflow", make_response("New")); - let entries = cache.entries.read().unwrap(); - let actor_entries = entries.get("actor").unwrap(); - assert_eq!(actor_entries.len(), IDEMPOTENCY_BUDGET_PER_ACTOR); - assert!(actor_entries.contains_key("k-overflow")); - } -} +#[path = "idempotency_test.rs"] +mod tests; diff --git a/crates/temper-server/src/idempotency_test.rs b/crates/temper-server/src/idempotency_test.rs new file mode 100644 index 000000000..557d862fd --- /dev/null +++ b/crates/temper-server/src/idempotency_test.rs @@ -0,0 +1,444 @@ +use super::*; +use crate::entity_actor::{EntityResponse, EntityState}; +use std::cell::Cell; +use std::collections::BTreeMap; +use std::future::Future; +use std::task::{Context, Poll, Waker}; + +fn drop_after_first_pending(future: F) +where + F: Future, +{ + let mut future = Box::pin(future); + let mut context = Context::from_waker(Waker::noop()); + assert!(matches!(future.as_mut().poll(&mut context), Poll::Pending)); +} + +fn make_response(status: &str) -> EntityResponse { + EntityResponse { + success: true, + state: EntityState { + entity_type: String::new(), + entity_id: String::new(), + status: status.to_string(), + item_count: 0, + counters: BTreeMap::new(), + booleans: BTreeMap::new(), + lists: BTreeMap::new(), + fields: serde_json::json!({}), + events: std::collections::VecDeque::new(), + total_event_count: 0, + events_since_snapshot: 0, + last_snapshot_sequence_nr: 0, + sequence_nr: 0, + processed_idempotency_keys: BTreeMap::new(), + }, + error: None, + custom_effects: vec![], + scheduled_actions: vec![], + spawn_requests: vec![], + spec_governed: true, + } +} + +#[test] +fn put_then_get_returns_cached() { + let cache = IdempotencyCache::new(); + let resp = make_response("Active"); + cache.put("Order:o1", "key-1", resp.clone()); + let cached = cache.get("Order:o1", "key-1"); + assert!(cached.is_some()); + assert_eq!(cached.unwrap().state.status, "Active"); +} + +#[test] +fn pending_effects_do_not_satisfy_protocol_cache_hit() { + let cache = IdempotencyCache::new(); + cache.put("Order:o1", "key-1", make_response("Active")); + + assert!(cache.get("Order:o1", "key-1").is_some()); + assert!( + cache + .get_after_effects_applied("Order:o1", "key-1") + .is_none() + ); + + assert!(cache.mark_effects_applied("Order:o1", "key-1")); + assert!( + cache + .get_after_effects_applied("Order:o1", "key-1") + .is_some() + ); +} + +#[test] +fn put_effects_applied_satisfies_protocol_cache_hit() { + let cache = IdempotencyCache::new(); + cache.put_effects_applied("Order:o1", "key-1", make_response("Active")); + + let cached = cache.get_after_effects_applied("Order:o1", "key-1"); + assert_eq!(cached.unwrap().state.status, "Active"); +} + +#[test] +fn unproved_actor_entry_conflicts_with_bound_action_claim() { + let cache = IdempotencyCache::new(); + cache.put("Order:o1", "key-1", make_response("Active")); + + assert!(matches!( + cache.lookup_bound_action_replay("Order:o1", "key-1", "request-a"), + BoundActionReplayLookup::Conflict + )); + assert!(matches!( + cache.claim_bound_action( + "Order:o1", + "key-1", + "request-a", + &serde_json::json!({"Reason": "a"}), + ), + BoundActionClaim::Conflict + )); +} + +#[test] +fn bound_action_claim_serializes_same_key_requests() { + let cache = IdempotencyCache::new(); + let params = serde_json::json!({"Reason": "a"}); + assert!(matches!( + cache.claim_bound_action("Order:o1", "key-1", "request-a", ¶ms), + BoundActionClaim::Claimed + )); + assert!(matches!( + cache.claim_bound_action("Order:o1", "key-1", "request-a", ¶ms), + BoundActionClaim::Pending + )); + assert!(matches!( + cache.claim_bound_action("Order:o1", "key-1", "request-b", ¶ms), + BoundActionClaim::Conflict + )); + + cache.put_bound_action_effects_applied( + "Order:o1", + "key-1", + make_response("Done"), + "request-a".to_string(), + params.clone(), + ); + assert!(matches!( + cache.claim_bound_action("Order:o1", "key-1", "request-a", ¶ms), + BoundActionClaim::Pending + )); + cache.fail_bound_action_hook("Order:o1", "key-1", "request-a"); + assert!(matches!( + cache.claim_bound_action("Order:o1", "key-1", "request-a", ¶ms), + BoundActionClaim::Match { .. } + )); +} + +#[test] +fn in_flight_bound_action_hook_is_not_evicted_by_the_actor_budget() { + let cache = IdempotencyCache::new(); + let actor_key = "Order:hook-eviction"; + let response = make_response("Done"); + cache.put_bound_action_effects_applied( + actor_key, + "000-hook-in-flight", + response.clone(), + "request-hook".to_string(), + serde_json::json!({"Reason": "one hook"}), + ); + + for index in 0..IDEMPOTENCY_BUDGET_PER_ACTOR { + cache.put_effects_applied(actor_key, &format!("z-fill-{index:04}"), response.clone()); + } + + assert!( + cache.complete_bound_action_hook( + actor_key, + "000-hook-in-flight", + "request-hook", + Some(serde_json::json!({"attempt": 1})), + ), + "a successful hook must retain its reserved cache entry while it is in flight" + ); +} + +fn fill_actor_with_protected_hooks(cache: &IdempotencyCache, actor_key: &str) { + let response = make_response("Done"); + for index in 0..IDEMPOTENCY_BUDGET_PER_ACTOR { + let key = format!("protected-{index:04}"); + cache.put_bound_action_effects_applied( + actor_key, + &key, + response.clone(), + format!("request-{index:04}"), + serde_json::json!({"index": index}), + ); + } +} + +#[test] +fn protected_entries_keep_claim_admission_within_actor_budget() { + let cache = IdempotencyCache::new(); + let actor_key = "Order:claim-saturation"; + fill_actor_with_protected_hooks(&cache, actor_key); + + let admission = cache.claim_bound_action( + actor_key, + "overflow", + "request-overflow", + &serde_json::json!({"Reason": "overflow"}), + ); + + let entries = cache.entries.read().unwrap(); + let actor_entries = entries.get(actor_key).unwrap(); + assert_eq!(actor_entries.len(), IDEMPOTENCY_BUDGET_PER_ACTOR); + assert!(!matches!(admission, BoundActionClaim::Claimed)); + assert!(!actor_entries.contains_key("overflow")); +} + +#[test] +fn protected_entries_keep_response_insertion_within_actor_budget() { + let cache = IdempotencyCache::new(); + let actor_key = "Order:response-saturation"; + fill_actor_with_protected_hooks(&cache, actor_key); + + assert!(!cache.put_bound_action_effects_applied( + actor_key, + "overflow", + make_response("Done"), + "request-overflow".to_string(), + serde_json::json!({"Reason": "overflow"}), + )); + + let entries = cache.entries.read().unwrap(); + let actor_entries = entries.get(actor_key).unwrap(); + assert_eq!(actor_entries.len(), IDEMPOTENCY_BUDGET_PER_ACTOR); + assert!(!actor_entries.contains_key("overflow")); +} + +#[test] +fn protected_entries_keep_active_claim_under_budget_pressure() { + let cache = IdempotencyCache::new(); + let actor_key = "Order:active-claim-saturation"; + assert!(matches!( + cache.claim_bound_action( + actor_key, + "active", + "request-active", + &serde_json::json!({"Reason": "active"}), + ), + BoundActionClaim::Claimed + )); + + let response = make_response("Done"); + for index in 0..(IDEMPOTENCY_BUDGET_PER_ACTOR - 1) { + let key = format!("protected-{index:04}"); + cache.put_bound_action_effects_applied( + actor_key, + &key, + response.clone(), + format!("request-{index:04}"), + serde_json::json!({"index": index}), + ); + } + + let admission = cache.claim_bound_action( + actor_key, + "overflow", + "request-overflow", + &serde_json::json!({"Reason": "overflow"}), + ); + let entries = cache.entries.read().unwrap(); + let actor_entries = entries.get(actor_key).unwrap(); + assert_eq!(actor_entries.len(), IDEMPOTENCY_BUDGET_PER_ACTOR); + assert!(actor_entries.contains_key("active")); + assert!(!actor_entries.contains_key("overflow")); + assert!(!matches!(admission, BoundActionClaim::Claimed)); +} + +#[test] +fn active_claim_does_not_age_out_before_its_owner_releases_it() { + let cache = IdempotencyCache::new(); + let actor_key = "Order:active-claim-ttl"; + let key = "active"; + let fingerprint = "request-active"; + let params = serde_json::json!({"Reason": "active"}); + assert!(matches!( + cache.claim_bound_action(actor_key, key, fingerprint, ¶ms), + BoundActionClaim::Claimed + )); + cache + .entries + .write() + .unwrap() + .get_mut(actor_key) + .unwrap() + .get_mut(key) + .unwrap() + .created_at = sim_now() - chrono::Duration::seconds(IDEMPOTENCY_TTL_SECS + 1); + + assert!(matches!( + cache.claim_bound_action(actor_key, key, fingerprint, ¶ms), + BoundActionClaim::Pending + )); +} + +#[test] +fn cancelling_dispatch_await_releases_raw_bound_action_claim() { + let cache = IdempotencyCache::new(); + let actor_key = "Order:cancelled-dispatch"; + let key = "active"; + let fingerprint = "request-active"; + let params = serde_json::json!({"Reason": "active"}); + assert!(matches!( + cache.claim_bound_action(actor_key, key, fingerprint, ¶ms), + BoundActionClaim::Claimed + )); + + drop_after_first_pending(async { + let _guard = cache.guard_bound_action_reservation(actor_key, key, fingerprint); + std::future::pending::<()>().await; + }); + + assert!(matches!( + cache.claim_bound_action(actor_key, key, fingerprint, ¶ms), + BoundActionClaim::Claimed + )); +} + +#[test] +fn cancelling_hook_await_releases_owner_and_pins_new_publication_debt() { + let cache = IdempotencyCache::new(); + let actor_key = "Order:cancelled-hook"; + let key = "active"; + let fingerprint = "request-active"; + assert!(cache.put_bound_action_effects_applied( + actor_key, + key, + make_response("Done"), + fingerprint.to_string(), + serde_json::json!({"Reason": "active"}), + )); + let publication_gated = Cell::new(false); + + drop_after_first_pending(async { + let _guard = + cache.guard_bound_action_hook(actor_key, key, fingerprint, || publication_gated.get()); + publication_gated.set(true); + std::future::pending::<()>().await; + }); + + let entries = cache.entries.read().unwrap(); + let entry = entries.get(actor_key).unwrap().get(key).unwrap(); + assert!(entry.publication_replay_pinned); + assert!(!entry.bound_action_hook_in_flight); + drop(entries); + assert!(matches!( + cache.lookup_bound_action_replay(actor_key, key, fingerprint), + BoundActionReplayLookup::Match { .. } + )); +} + +#[test] +fn publication_replay_pin_survives_ttl_until_completed_and_unpinned() { + let cache = IdempotencyCache::new(); + let actor = "Order:o1"; + let key = "publication-key"; + let fingerprint = "request-a"; + let params = serde_json::json!({"Reason": "publish"}); + cache.put_bound_action_effects_applied( + actor, + key, + make_response("Done"), + fingerprint.to_string(), + params, + ); + assert!(cache.pin_bound_action_replay(actor, key, fingerprint)); + cache.fail_bound_action_hook(actor, key, fingerprint); + cache + .entries + .write() + .unwrap() + .get_mut(actor) + .unwrap() + .get_mut(key) + .unwrap() + .created_at = sim_now() - chrono::Duration::seconds(IDEMPOTENCY_TTL_SECS + 1); + + assert!(matches!( + cache.lookup_bound_action_replay(actor, key, fingerprint), + BoundActionReplayLookup::Match { .. } + )); + + // An in-flight retry is not enough to discharge the outcome-ambiguous + // publication debt: cancellation can still leave the hook incomplete. + cache.unpin_bound_action_replay(actor, key, fingerprint); + assert!( + cache + .entries + .read() + .unwrap() + .get(actor) + .unwrap() + .get(key) + .unwrap() + .publication_replay_pinned + ); + + assert!(cache.complete_bound_action_hook(actor, key, fingerprint, None)); + assert!( + !cache + .entries + .read() + .unwrap() + .get(actor) + .unwrap() + .get(key) + .unwrap() + .publication_replay_pinned + ); + cache + .entries + .write() + .unwrap() + .get_mut(actor) + .unwrap() + .get_mut(key) + .unwrap() + .created_at = sim_now() - chrono::Duration::seconds(IDEMPOTENCY_TTL_SECS + 1); + assert!(matches!( + cache.lookup_bound_action_replay(actor, key, fingerprint), + BoundActionReplayLookup::Miss + )); +} + +#[test] +fn get_missing_returns_none() { + let cache = IdempotencyCache::new(); + assert!(cache.get("Order:o1", "no-such-key").is_none()); +} + +#[test] +fn different_actors_isolated() { + let cache = IdempotencyCache::new(); + cache.put("Order:o1", "key-1", make_response("A")); + cache.put("Order:o2", "key-1", make_response("B")); + assert_eq!(cache.get("Order:o1", "key-1").unwrap().state.status, "A"); + assert_eq!(cache.get("Order:o2", "key-1").unwrap().state.status, "B"); +} + +#[test] +fn budget_evicts_oldest() { + let cache = IdempotencyCache::new(); + // Fill to budget + for i in 0..IDEMPOTENCY_BUDGET_PER_ACTOR { + cache.put("actor", &format!("k-{i}"), make_response("S")); + } + // One more should evict the oldest + cache.put("actor", "k-overflow", make_response("New")); + let entries = cache.entries.read().unwrap(); + let actor_entries = entries.get("actor").unwrap(); + assert_eq!(actor_entries.len(), IDEMPOTENCY_BUDGET_PER_ACTOR); + assert!(actor_entries.contains_key("k-overflow")); +} diff --git a/crates/temper-server/src/key_index.rs b/crates/temper-server/src/key_index.rs index f5b69990a..43b3efe4b 100644 --- a/crates/temper-server/src/key_index.rs +++ b/crates/temper-server/src/key_index.rs @@ -13,6 +13,7 @@ use sha2::{Digest, Sha256}; use temper_jit::table::types::DeclaredKey; +use temper_runtime::persistence::EntityKeyRow; /// Separates `key_name` from the value list. const UNIT_SEP: u8 = 0x1F; @@ -27,15 +28,77 @@ const TAG_BOOL: u8 = b'B'; /// `ParentId`), tagged so it can never collide with the empty string. const TAG_NULL: u8 = b'0'; -/// The stable signature of a type's declared key-set: the sorted, comma-joined key -/// NAMES. Recorded in the key-index backfill watermark and compared on read, so that -/// declaring an ADDITIONAL key (a changed signature) re-keys the type instead of being -/// treated as already complete, and a keyed read only trusts absence once the full -/// current key-set is backfilled (ARN-68). Deterministic (sorted, no map iteration). +/// Changes whenever key-row derivation/reconciliation semantics change. Including +/// it in the durable coverage signature forces a one-time authoritative repair of +/// rows created under an older contract (ADR-0192). +const KEY_INDEX_DERIVATION_VERSION: &str = "v3"; + +/// The stable signature of the key derivation contract and a type's declared key-set: +/// a version prefix plus every sorted declaration's name and ordered property list. +/// Length prefixes keep the encoding unambiguous even if future identifiers contain +/// punctuation. Recorded in the backfill watermark and compared on read, so a name, +/// property, property-order, or semantics change re-keys the type. Deterministic +/// (sorted declarations, declared property order, no map iteration). pub fn declared_key_set_signature(keys: &[DeclaredKey]) -> String { - let mut names: Vec<&str> = keys.iter().map(|key| key.name.as_str()).collect(); - names.sort(); - names.join(",") + let mut declarations = keys.iter().collect::>(); + declarations.sort_by(|left, right| { + left.name + .cmp(&right.name) + .then_with(|| left.properties.cmp(&right.properties)) + }); + + let mut encoded = String::from(KEY_INDEX_DERIVATION_VERSION); + for key in declarations { + encoded.push('|'); + encoded.push_str(&key.name.len().to_string()); + encoded.push(':'); + encoded.push_str(&key.name); + encoded.push('['); + for property in &key.properties { + encoded.push_str(&property.len().to_string()); + encoded.push(':'); + encoded.push_str(property); + } + encoded.push(']'); + } + encoded +} + +/// Contract token carried by a spec-governed writer. The stable signature +/// drives coverage; the activation epoch rejects a delayed writer even when a +/// hot swap cycles back to byte-identical declarations. +pub fn declared_key_write_contract(table: &temper_jit::table::TransitionTable) -> String { + let key_set = declared_key_set_signature(&table.keys); + if table.key_contract_activation_epoch == 0 { + key_set + } else { + temper_runtime::persistence::encode_activated_key_contract( + &key_set, + table.key_contract_activation_epoch, + ) + } +} + +/// Derive an entity's complete current declared-key row set from authoritative +/// post-transition fields. `index_entity = false` produces the exact empty set for +/// a tombstone. Shared by ordinary actor appends and atomic composite batches so the +/// two write paths cannot diverge on null, rename, or delete semantics (ADR-0192). +pub(crate) fn derive_entity_key_rows( + keys: &[DeclaredKey], + fields: &serde_json::Value, + index_entity: bool, +) -> Vec { + let Some(field_map) = fields.as_object().filter(|_| index_entity) else { + return Vec::new(); + }; + keys.iter() + .filter_map(|key| { + canonical_key_hash(&key.name, &key.properties, field_map).map(|key_hash| EntityKeyRow { + key_name: key.name.clone(), + key_hash, + }) + }) + .collect() } /// Canonical `key_hash` for a declared key's values, or `None` when the key is diff --git a/crates/temper-server/src/observe/mod_test.rs b/crates/temper-server/src/observe/mod_test.rs index b5bf6abd0..ee8163713 100644 --- a/crates/temper-server/src/observe/mod_test.rs +++ b/crates/temper-server/src/observe/mod_test.rs @@ -475,6 +475,116 @@ async fn wasm_upload_accepts_json_base64_body() { assert_eq!(json["size_bytes"], VALID_EMPTY_WASM.len()); } +#[tokio::test] +async fn wasm_upload_cannot_cross_a_stable_tenant_generation() { + let state = test_state_with_turso().await; + let tenant = TenantId::default(); + let app = build_app_with_state(state.clone()); + let stable_reader = state.begin_tenant_request(&tenant).await; + + let blocked = app + .clone() + .oneshot( + Request::post("/api/wasm/modules/generation_fenced") + .header("X-Tenant-Id", "default") + .header("X-Temper-Principal-Kind", "admin") + .body(Body::from(VALID_EMPTY_WASM.to_vec())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(blocked.status(), StatusCode::SERVICE_UNAVAILABLE); + assert!( + state + .wasm_module_registry + .read() + .expect("WASM registry lock poisoned") + .get_hash(&tenant, "generation_fenced") + .is_none() + ); + + drop(stable_reader); + let published = app + .oneshot( + Request::post("/api/wasm/modules/generation_fenced") + .header("X-Tenant-Id", "default") + .header("X-Temper-Principal-Kind", "admin") + .body(Body::from(VALID_EMPTY_WASM.to_vec())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(published.status(), StatusCode::OK); + assert!( + state + .wasm_module_registry + .read() + .expect("WASM registry lock poisoned") + .get_hash(&tenant, "generation_fenced") + .is_some() + ); +} + +#[tokio::test] +async fn wasm_read_cannot_observe_an_armed_tenant_generation() { + let state = test_state_with_turso().await; + let tenant = TenantId::default(); + let app = build_app_with_state(state.clone()); + let uploaded = app + .clone() + .oneshot( + Request::post("/api/wasm/modules/read_generation_fenced") + .header("X-Tenant-Id", "default") + .header("X-Temper-Principal-Kind", "admin") + .body(Body::from(VALID_EMPTY_WASM.to_vec())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(uploaded.status(), StatusCode::OK); + + let mut publication = state + .begin_spec_publication(&tenant) + .await + .expect("acquire publication writer"); + let intent = ServerState::spec_publication_intent( + "observe-wasm-read-generation-test", + [("module", b"read_generation_fenced".as_slice())], + ); + state + .arm_spec_publication(&mut publication, &tenant, &intent) + .expect("arm publication writer"); + + let blocked = app + .clone() + .oneshot( + Request::get("/observe/wasm/modules/read_generation_fenced") + .header("X-Tenant-Id", "default") + .header("X-Temper-Principal-Kind", "admin") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(blocked.status(), StatusCode::SERVICE_UNAVAILABLE); + + state + .complete_spec_publication(&mut publication, &tenant) + .expect("complete publication generation"); + drop(publication); + let visible = app + .oneshot( + Request::get("/observe/wasm/modules/read_generation_fenced") + .header("X-Tenant-Id", "default") + .header("X-Temper-Principal-Kind", "admin") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(visible.status(), StatusCode::OK); +} + #[tokio::test] async fn approved_wasm_upload_decision_allows_agent_retry() { let state = test_state_with_turso().await; @@ -1982,6 +2092,138 @@ async fn test_load_dir_registers_specs() { ); } +#[tokio::test] +async fn test_load_dir_policy_survives_canonical_generation_recovery() { + let state = test_state_with_turso().await; + let tenant = "load-dir-policy-recovery"; + let primary = r#"permit(principal, action == Action::"primary_only", resource);"#; + let bundled = r#"permit(principal, action == Action::"bundled_only", resource);"#; + state + .policy_store() + .expect("policy store") + .save_policy(tenant, "primary", primary, "test") + .await + .expect("seed canonical primary"); + state + .authz + .reload_tenant_policies(tenant, primary) + .expect("activate primary"); + state + .tenant_policies + .write() + .unwrap() + .insert(tenant.to_string(), primary.to_string()); + + let app = Router::new() + .nest("/observe", build_observe_router()) + .nest("/api", crate::api::build_api_router()) + .with_state(state.clone()); + let specs_dir = + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test-fixtures/specs"); + let body = serde_json::json!({ + "tenant": tenant, + "specs_dir": specs_dir.to_str().unwrap(), + "merge": true, + "cedar_policies": bundled, + }); + let response = app + .oneshot( + Request::post("/api/specs/load-dir") + .header("Content-Type", "application/json") + .body(Body::from(serde_json::to_string(&body).unwrap())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + + let rows = state + .policy_store() + .expect("policy store") + .load_policies_for_tenant(tenant) + .await + .expect("load canonical generation"); + assert!(rows.iter().any(|row| row.policy_id == "primary")); + assert!(rows.iter().any(|row| { + row.policy_id.starts_with("observe-load-dir-") + && row.created_by.starts_with("observe:load-dir:") + && row.cedar_text.contains("bundled_only") + })); + + state + .authz + .reload_tenant_policies_named(tenant, &[]) + .expect("clear live generation before recovery"); + crate::authz::load_and_activate_tenant_policies(&state, tenant).await; + let recovered = state + .authz + .get_tenant_policy_text(tenant) + .expect("recover canonical generation"); + assert!(recovered.contains("primary_only")); + assert!(recovered.contains("bundled_only")); +} + +#[tokio::test] +async fn test_load_dir_replaces_its_owned_policy_in_live_and_recovered_generations() { + let state = test_state_with_turso().await; + let tenant = "load-dir-policy-replacement"; + let allow_a = r#"permit(principal, action == Action::"allow_a", resource);"#; + let allow_b = r#"permit(principal, action == Action::"allow_b", resource);"#; + let app = Router::new() + .nest("/observe", build_observe_router()) + .nest("/api", crate::api::build_api_router()) + .with_state(state.clone()); + let specs_dir = + std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../test-fixtures/specs"); + let request = |cedar_policies: &str| { + let body = serde_json::json!({ + "tenant": tenant, + "specs_dir": specs_dir.to_str().unwrap(), + "merge": true, + "cedar_policies": cedar_policies, + }); + Request::post("/api/specs/load-dir") + .header("Content-Type", "application/json") + .body(Body::from(serde_json::to_string(&body).unwrap())) + .unwrap() + }; + + assert_eq!( + app.clone() + .oneshot(request(allow_a)) + .await + .unwrap() + .status(), + StatusCode::OK + ); + assert_eq!( + app.oneshot(request(allow_b)).await.unwrap().status(), + StatusCode::OK + ); + + let live = state + .authz + .get_tenant_policy_text(tenant) + .expect("live replacement policy"); + assert!(!live.contains("allow_a"), "replaced grant remained live"); + assert!(live.contains("allow_b")); + + state + .authz + .reload_tenant_policies_named(tenant, &[]) + .expect("clear live generation before recovery"); + crate::authz::load_and_activate_tenant_policies(&state, tenant).await; + let recovered = state + .authz + .get_tenant_policy_text(tenant) + .expect("recovered replacement policy"); + assert!( + !recovered.contains("allow_a"), + "replaced grant returned after recovery" + ); + assert!(recovered.contains("allow_b")); +} + #[tokio::test] async fn test_load_dir_missing_dir_returns_error() { let system = ActorSystem::new("test-load-dir-missing"); diff --git a/crates/temper-server/src/observe/specs/load_dir.rs b/crates/temper-server/src/observe/specs/load_dir.rs index ca3500398..a986dbd46 100644 --- a/crates/temper-server/src/observe/specs/load_dir.rs +++ b/crates/temper-server/src/observe/specs/load_dir.rs @@ -1,20 +1,26 @@ use axum::extract::State; use axum::http::StatusCode; use axum::response::Json; +use sha2::{Digest, Sha256}; use temper_spec::automaton::LintSeverity; -use temper_spec::cross_invariant::{ - CrossInvariantLintSeverity, lint_cross_invariants, parse_cross_invariants, -}; +use temper_spec::cross_invariant::CrossInvariantLintSeverity; use tracing::instrument; use super::super::specs_helpers::{ - build_ndjson_response, cross_lint_ndjson_line, lint_loaded_specs, lint_ndjson_line, - to_pascal_case, + build_ndjson_response, cross_lint_ndjson_line, lint_ndjson_line, }; use super::types::LoadDirRequest; use super::verification_stream::build_verification_stream_response; +use crate::platform_store::{ + PolicyEntryPublication, PolicyGenerationPublication, SpecPublication, SpecPublicationMode, + TenantConstraintsPublication, TenantPolicyPublication, +}; use crate::state::ServerState; +mod source_loading; + +use source_loading::{LoadedSpecSources, load_spec_sources}; + /// POST /api/specs/load-dir -- hot-load specs from a directory into the running server./// /// Reads CSDL and IOA files from `specs_dir`, registers them under `tenant`, /// emits design-time SSE events for each entity, and spawns background @@ -33,128 +39,18 @@ pub(crate) async fn handle_load_dir( )); } - // Read CSDL model - let csdl_path = specs_path.join("model.csdl.xml"); - if !csdl_path.exists() { - return Err(( - StatusCode::BAD_REQUEST, - format!("CSDL model not found at {}", csdl_path.display()), - )); - } - - let csdl_xml = std::fs::read_to_string(&csdl_path).map_err(|e| { - // determinism-ok: HTTP handler reads spec files - ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to read CSDL: {e}"), - ) - })?; - let csdl = temper_spec::csdl::parse_csdl(&csdl_xml).map_err(|e| { - ( - StatusCode::BAD_REQUEST, - format!("Failed to parse CSDL: {e}"), - ) - })?; - - // Read all *.ioa.toml files - let mut ioa_sources: std::collections::BTreeMap = - std::collections::BTreeMap::new(); - let entries = std::fs::read_dir(specs_path).map_err(|e| { - // determinism-ok: HTTP handler reads spec directory - ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to read specs directory: {e}"), - ) - })?; - for entry in entries { - let entry = entry.map_err(|e| { - ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to read directory entry: {e}"), - ) - })?; - let path = entry.path(); - let file_name = path - .file_name() - .and_then(|n| n.to_str()) - .unwrap_or_default(); - if file_name.ends_with(".ioa.toml") { - let entity_name = file_name.strip_suffix(".ioa.toml").unwrap_or_default(); - let entity_name = to_pascal_case(entity_name); - let source = std::fs::read_to_string(&path).map_err(|e| { - // determinism-ok: HTTP handler reads spec files - ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to read {}: {e}", path.display()), - ) - })?; - ioa_sources.insert(entity_name, source); - } - } - - if ioa_sources.is_empty() { - return Err(( - StatusCode::BAD_REQUEST, - "No .ioa.toml files found in specs directory".to_string(), - )); - } - - let legacy_reactions_path = specs_path.join("reactions.toml"); - if legacy_reactions_path.exists() { - return Err(( - StatusCode::BAD_REQUEST, - format!( - "Legacy {} is no longer supported; migrate to inline [[action.triggers]]", - legacy_reactions_path.display() - ), - )); - } - - // Optional cross-invariants.toml. - let cross_invariants_toml = { - let path = specs_path.join("cross-invariants.toml"); - if path.exists() { - Some(std::fs::read_to_string(&path).map_err(|e| { - // determinism-ok: HTTP handler reads cross-invariants - ( - StatusCode::INTERNAL_SERVER_ERROR, - format!("Failed to read {}: {e}", path.display()), - ) - })?) - } else { - None - } - }; - - let lint_findings = lint_loaded_specs(&csdl, &ioa_sources)?; - let cross_lint_findings = if let Some(source) = cross_invariants_toml.as_deref() { - let spec = parse_cross_invariants(source).map_err(|e| { - ( - StatusCode::BAD_REQUEST, - format!("Failed to parse cross-invariants.toml: {e}"), - ) - })?; - lint_cross_invariants(&spec) - } else { - Vec::new() - }; - - let ioa_lint_errors = lint_findings - .iter() - .filter(|f| matches!(f.severity, LintSeverity::Error)) - .count(); - let ioa_lint_warnings = lint_findings - .iter() - .filter(|f| matches!(f.severity, LintSeverity::Warning)) - .count(); - let cross_lint_errors = cross_lint_findings - .iter() - .filter(|f| matches!(f.severity, CrossInvariantLintSeverity::Error)) - .count(); - let cross_lint_warnings = cross_lint_findings - .iter() - .filter(|f| matches!(f.severity, CrossInvariantLintSeverity::Warning)) - .count(); + let LoadedSpecSources { + csdl_xml, + csdl, + ioa_sources, + cross_invariants_toml, + lint_findings, + cross_lint_findings, + ioa_lint_errors, + ioa_lint_warnings, + cross_lint_errors, + cross_lint_warnings, + } = load_spec_sources(specs_path)?; let lint_errors = ioa_lint_errors + cross_lint_errors; let lint_warnings = ioa_lint_warnings + cross_lint_warnings; @@ -184,29 +80,256 @@ pub(crate) async fn handle_load_dir( return build_ndjson_response(StatusCode::BAD_REQUEST, lines); } - // Persist loaded specs first when Postgres is configured. - let csdl_xml_for_db = csdl_xml.clone(); - for (entity_type, ioa_source) in &ioa_sources { - state - .upsert_spec_source(&body.tenant, entity_type, ioa_source, &csdl_xml_for_db) - .await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e))?; - } - state - .upsert_tenant_constraints(&body.tenant, cross_invariants_toml.as_deref()) - .await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e))?; - - // Register into shared registry after persistence succeeds. let ioa_pairs: Vec<(&str, &str)> = ioa_sources .iter() .map(|(k, v)| (k.as_str(), v.as_str())) .collect(); + let tenant_id = temper_runtime::tenant::TenantId::new(&body.tenant); + let mut publication_guard = state + .begin_spec_publication(&tenant_id) + .await + .map_err(|error| (StatusCode::SERVICE_UNAVAILABLE, error))?; + let incoming_cedar_policy = body + .cedar_policies + .as_deref() + .map(str::trim) + .filter(|source| !source.is_empty()) + .map(str::to_string); + let mut owned_policy_entries = Vec::<(String, String, String)>::new(); + let mut policy_owner = None; + let complete_cedar_policy = match incoming_cedar_policy.as_deref() { + Some(incoming) => { + incoming + .parse::() + .map_err(|error| { + ( + StatusCode::BAD_REQUEST, + format!("Bundled Cedar policies failed to parse: {error}"), + ) + })?; + let cached = state + .tenant_policies + .read() + .map_err(|error| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("tenant policy lock poisoned: {error}"), + ) + })? + .get(&body.tenant) + .cloned() + .unwrap_or_default(); + let source_digest = format!("{:x}", Sha256::digest(body.specs_dir.as_bytes())); + let owner = format!("observe:load-dir:{source_digest}"); + let policy_id = format!("observe-load-dir-{source_digest}"); + let complete = if let Some(store) = state.policy_store() { + let rows = store + .load_policies_for_tenant(&body.tenant) + .await + .map_err(|error| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to load durable Cedar policy generation: {error}"), + ) + })?; + let had_canonical_rows = !rows.is_empty(); + let mut complete_rows = rows + .into_iter() + .filter(|row| row.created_by != owner) + .map(|row| (row.policy_id, row.cedar_text, row.created_by, row.enabled)) + .collect::>(); + if !had_canonical_rows { + let legacy = store + .load_policy_compatibility_text(&body.tenant) + .await + .map_err(|error| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to load legacy Cedar policy generation: {error}"), + ) + })? + .unwrap_or(cached); + if !legacy.trim().is_empty() { + let primary = ( + "primary".to_string(), + legacy.trim().to_string(), + "legacy-migration".to_string(), + ); + complete_rows.push(( + primary.0.clone(), + primary.1.clone(), + primary.2.clone(), + true, + )); + owned_policy_entries.push(primary); + } + } + complete_rows.push((policy_id.clone(), incoming.to_string(), owner.clone(), true)); + complete_rows.sort_by(|left, right| left.0.cmp(&right.0)); + complete_rows + .into_iter() + .filter(|(_, _, _, enabled)| *enabled) + .map(|(_, cedar_text, _, _)| cedar_text) + .collect::>() + .join("\n") + } else { + super::load_inline::merge_inline_cedar_policy_text(&cached, incoming) + }; + owned_policy_entries.push((policy_id, incoming.to_string(), owner.clone())); + policy_owner = Some(owner); + complete + .parse::() + .map_err(|error| { + ( + StatusCode::BAD_REQUEST, + format!("Complete tenant Cedar policies failed to parse: {error}"), + ) + })?; + Some(complete) + } + None => None, + }; + let incoming_types = ioa_sources + .keys() + .map(String::as_str) + .collect::>(); + let mut removed_entity_types = if body.merge { + std::collections::BTreeSet::new() + } else { + state + .registry + .read() + .expect("registry lock poisoned") + .entity_types(&tenant_id) + .into_iter() + .filter(|entity_type| !incoming_types.contains(entity_type)) + .map(str::to_string) + .collect::>() + }; + + // One durable transaction publishes every source, replace-mode deletion, + // and the matching cross-invariant generation before any runtime fence moves. + let publications = ioa_sources + .iter() + .map(|(entity_type, ioa_source)| { + ( + entity_type, + ioa_source, + temper_store_turso::spec_content_hash(ioa_source), + ) + }) + .collect::>(); + let publication_refs = publications + .iter() + .map(|(entity_type, ioa_source, content_hash)| SpecPublication { + entity_type, + ioa_source, + csdl_xml: &csdl_xml, + content_hash, + }) + .collect::>(); + let mut intent_components = vec![ + ("csdl".to_string(), csdl_xml.as_bytes().to_vec()), + ( + "mode".to_string(), + if body.merge { "merge" } else { "replace" } + .as_bytes() + .to_vec(), + ), + ]; + let constraints_intent = match (body.merge, cross_invariants_toml.as_deref()) { + (true, None) => b"preserve".to_vec(), + (_, Some(source)) => source.as_bytes().to_vec(), + (_, None) => b"delete".to_vec(), + }; + intent_components.push(("constraints".to_string(), constraints_intent)); + intent_components.push(( + "policy".to_string(), + complete_cedar_policy + .as_deref() + .map(str::as_bytes) + .unwrap_or(b"preserve") + .to_vec(), + )); + intent_components.extend(ioa_sources.iter().map(|(entity_type, ioa_source)| { + ( + format!("spec:{entity_type}"), + ioa_source.as_bytes().to_vec(), + ) + })); + let intent_refs = intent_components + .iter() + .map(|(name, value)| (name.as_str(), value.as_slice())) + .collect::>(); + let publication_intent = ServerState::spec_publication_intent("load-specs-dir", intent_refs); + state + .arm_spec_publication(&mut publication_guard, &tenant_id, &publication_intent) + .map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error))?; + if let Some(store) = state + .storage_stack + .as_ref() + .and_then(|stack| stack.platform.clone()) + { + let policy_entry_refs = owned_policy_entries + .iter() + .map( + |(policy_id, cedar_text, created_by)| PolicyEntryPublication { + policy_id, + cedar_text, + created_by, + }, + ) + .collect::>(); + let policy_generation = + policy_owner + .as_deref() + .map(|policy_owner| PolicyGenerationPublication { + policy_owner, + policy_entries: &policy_entry_refs, + }); + let constraints = if body.merge && cross_invariants_toml.is_none() { + TenantConstraintsPublication::Preserve + } else { + TenantConstraintsPublication::Replace(cross_invariants_toml.as_deref()) + }; + let durable_removed = store + .publish_specs( + &body.tenant, + &publication_refs, + if body.merge { + SpecPublicationMode::Merge + } else { + SpecPublicationMode::Replace + }, + constraints, + complete_cedar_policy + .as_deref() + .map(TenantPolicyPublication::Replace) + .unwrap_or(TenantPolicyPublication::Preserve), + None, + policy_generation, + &[], + ) + .await + .map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error))?; + removed_entity_types.extend(durable_removed); + } + let removed_entity_types = removed_entity_types.into_iter().collect::>(); + + let mut cutover = state + .prepare_key_index_contracts_for_spec_activation_with_removals( + &publication_guard, + &tenant_id, + &ioa_pairs, + &removed_entity_types, + ) + .await + .map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error))?; { let mut registry = state.registry.write().unwrap(); // ci-ok: infallible lock registry - .try_register_tenant_with_reactions_and_constraints( + .try_register_tenant_with_reactions_constraints_and_key_epochs( body.tenant.as_str(), csdl, csdl_xml, @@ -214,6 +337,7 @@ pub(crate) async fn handle_load_dir( Vec::new(), cross_invariants_toml.clone(), body.merge, + &cutover.activation_epochs, ) .map_err(|e| { ( @@ -222,7 +346,35 @@ pub(crate) async fn handle_load_dir( ) })?; } + state + .finish_key_index_contract_activation(&mut publication_guard, &tenant_id, &mut cutover) + .await + .map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error))?; state.rebuild_reaction_dispatcher(); + if let Some(policy) = complete_cedar_policy { + state + .authz + .reload_tenant_policies(&body.tenant, &policy) + .map_err(|error| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to activate bundled Cedar policies: {error}"), + ) + })?; + state + .tenant_policies + .write() + .map_err(|error| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("tenant policy lock poisoned: {error}"), + ) + })? + .insert(body.tenant.clone(), policy); + } + state + .complete_spec_publication_retry(&mut publication_guard, &tenant_id) + .map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error))?; if !state.data_dir.as_os_str().is_empty() { let registry_path = state.data_dir.join("specs-registry.json"); diff --git a/crates/temper-server/src/observe/specs/load_dir/source_loading.rs b/crates/temper-server/src/observe/specs/load_dir/source_loading.rs new file mode 100644 index 000000000..8f8b076d5 --- /dev/null +++ b/crates/temper-server/src/observe/specs/load_dir/source_loading.rs @@ -0,0 +1,160 @@ +use std::collections::BTreeMap; +use std::path::Path; + +use axum::http::StatusCode; +use temper_spec::automaton::LintSeverity; +use temper_spec::cross_invariant::{ + CrossInvariantLintFinding, CrossInvariantLintSeverity, lint_cross_invariants, + parse_cross_invariants, +}; +use temper_spec::csdl::CsdlDocument; + +use super::super::super::specs_helpers::{EntityLintFinding, lint_loaded_specs, to_pascal_case}; + +pub(super) struct LoadedSpecSources { + pub(super) csdl_xml: String, + pub(super) csdl: CsdlDocument, + pub(super) ioa_sources: BTreeMap, + pub(super) cross_invariants_toml: Option, + pub(super) lint_findings: Vec, + pub(super) cross_lint_findings: Vec, + pub(super) ioa_lint_errors: usize, + pub(super) ioa_lint_warnings: usize, + pub(super) cross_lint_errors: usize, + pub(super) cross_lint_warnings: usize, +} + +pub(super) fn load_spec_sources( + specs_path: &Path, +) -> Result { + let csdl_path = specs_path.join("model.csdl.xml"); + if !csdl_path.exists() { + return Err(( + StatusCode::BAD_REQUEST, + format!("CSDL model not found at {}", csdl_path.display()), + )); + } + + let csdl_xml = std::fs::read_to_string(&csdl_path).map_err(|e| { + // determinism-ok: HTTP handler reads spec files + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to read CSDL: {e}"), + ) + })?; + let csdl = temper_spec::csdl::parse_csdl(&csdl_xml).map_err(|e| { + ( + StatusCode::BAD_REQUEST, + format!("Failed to parse CSDL: {e}"), + ) + })?; + + let mut ioa_sources = BTreeMap::new(); + let entries = std::fs::read_dir(specs_path).map_err(|e| { + // determinism-ok: HTTP handler reads spec directory + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to read specs directory: {e}"), + ) + })?; + for entry in entries { + let entry = entry.map_err(|e| { + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to read directory entry: {e}"), + ) + })?; + let path = entry.path(); + let file_name = path + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or_default(); + if file_name.ends_with(".ioa.toml") { + let entity_name = file_name.strip_suffix(".ioa.toml").unwrap_or_default(); + let entity_name = to_pascal_case(entity_name); + let source = std::fs::read_to_string(&path).map_err(|e| { + // determinism-ok: HTTP handler reads spec files + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to read {}: {e}", path.display()), + ) + })?; + ioa_sources.insert(entity_name, source); + } + } + if ioa_sources.is_empty() { + return Err(( + StatusCode::BAD_REQUEST, + "No .ioa.toml files found in specs directory".to_string(), + )); + } + + let legacy_reactions_path = specs_path.join("reactions.toml"); + if legacy_reactions_path.exists() { + return Err(( + StatusCode::BAD_REQUEST, + format!( + "Legacy {} is no longer supported; migrate to inline [[action.triggers]]", + legacy_reactions_path.display() + ), + )); + } + + let cross_invariants_toml = { + let path = specs_path.join("cross-invariants.toml"); + if path.exists() { + Some(std::fs::read_to_string(&path).map_err(|e| { + // determinism-ok: HTTP handler reads cross-invariants + ( + StatusCode::INTERNAL_SERVER_ERROR, + format!("Failed to read {}: {e}", path.display()), + ) + })?) + } else { + None + } + }; + + let lint_findings = lint_loaded_specs(&csdl, &ioa_sources)?; + let cross_lint_findings = if let Some(source) = cross_invariants_toml.as_deref() { + let spec = parse_cross_invariants(source).map_err(|e| { + ( + StatusCode::BAD_REQUEST, + format!("Failed to parse cross-invariants.toml: {e}"), + ) + })?; + lint_cross_invariants(&spec) + } else { + Vec::new() + }; + + let ioa_lint_errors = lint_findings + .iter() + .filter(|f| matches!(f.severity, LintSeverity::Error)) + .count(); + let ioa_lint_warnings = lint_findings + .iter() + .filter(|f| matches!(f.severity, LintSeverity::Warning)) + .count(); + let cross_lint_errors = cross_lint_findings + .iter() + .filter(|f| matches!(f.severity, CrossInvariantLintSeverity::Error)) + .count(); + let cross_lint_warnings = cross_lint_findings + .iter() + .filter(|f| matches!(f.severity, CrossInvariantLintSeverity::Warning)) + .count(); + + Ok(LoadedSpecSources { + csdl_xml, + csdl, + ioa_sources, + cross_invariants_toml, + lint_findings, + cross_lint_findings, + ioa_lint_errors, + ioa_lint_warnings, + cross_lint_errors, + cross_lint_warnings, + }) +} diff --git a/crates/temper-server/src/observe/specs/load_inline.rs b/crates/temper-server/src/observe/specs/load_inline.rs index 0eb24abf3..2da4704a6 100644 --- a/crates/temper-server/src/observe/specs/load_inline.rs +++ b/crates/temper-server/src/observe/specs/load_inline.rs @@ -234,41 +234,14 @@ pub(crate) async fn handle_load_inline( // Delegate to load-dir logic with merge=true so agent-submitted specs // are added to the existing tenant config instead of replacing it. - let cedar_policies = body.cedar_policies.clone(); let dir_request = LoadDirRequest { tenant: tenant.clone(), specs_dir: specs_root.to_string_lossy().to_string(), merge: true, + cedar_policies: body.cedar_policies.clone(), }; let result = handle_load_dir(State(state.clone()), Json(dir_request)).await; - if result.is_ok() - && let Some(ref cedar_text) = cedar_policies - && !cedar_text.trim().is_empty() - { - if let Err(e) = cedar_text.parse::() { - tracing::warn!(error = %e, "bundled Cedar policies failed to parse, skipping"); - } else { - // Update the in-memory text cache. - if let Ok(mut policies) = state.tenant_policies.write() { - let entry = policies.entry(tenant.clone()).or_default(); - *entry = merge_inline_cedar_policy_text(entry, cedar_text); - } - // Reload the per-tenant Cedar policy set. - let full_text = state - .tenant_policies - .read() - .ok() - .and_then(|p| p.get(&tenant).cloned()) - .unwrap_or_default(); - if let Err(e) = state.authz.reload_tenant_policies(&tenant, &full_text) { - tracing::error!(error = %e, "failed to reload policies with bundled Cedar"); - } else { - tracing::info!(tenant = %tenant, "bundled Cedar policies loaded successfully"); - } - } - } - if result.is_ok() { let warning_context = build_adr_warning_context(&state, &body, &tenant).await; for entity_name in &entity_names { @@ -391,7 +364,7 @@ fn resolve_inline_specs_root( }) } -fn merge_inline_cedar_policy_text(existing: &str, incoming: &str) -> String { +pub(super) fn merge_inline_cedar_policy_text(existing: &str, incoming: &str) -> String { let mut policy_text = existing.trim_end().to_string(); let incoming = incoming.trim(); if incoming.is_empty() || policy_text.contains(incoming) { diff --git a/crates/temper-server/src/observe/specs/types.rs b/crates/temper-server/src/observe/specs/types.rs index 9747b90ca..4f9003825 100644 --- a/crates/temper-server/src/observe/specs/types.rs +++ b/crates/temper-server/src/observe/specs/types.rs @@ -12,6 +12,10 @@ pub(crate) struct LoadDirRequest { /// wipe platform entity types. #[serde(default)] pub(crate) merge: bool, + /// Optional Cedar policy text merged into the tenant's complete policy and + /// committed/activated in the same guarded runtime generation. + #[serde(default)] + pub(crate) cedar_policies: Option, } /// Request body for POST /api/specs/load-inline. diff --git a/crates/temper-server/src/observe/wasm.rs b/crates/temper-server/src/observe/wasm.rs index 43e61585c..8667a1beb 100644 --- a/crates/temper-server/src/observe/wasm.rs +++ b/crates/temper-server/src/observe/wasm.rs @@ -2,14 +2,14 @@ //! //! Upload, download, delete, and list WASM integration modules. -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; -use axum::extract::Path; -use axum::extract::{Query, State}; +use axum::extract::{Path, State}; use axum::http::{HeaderMap, StatusCode}; use axum::response::Json; use base64::Engine; use serde::{Deserialize, Serialize}; +use temper_runtime::tenant::TenantId; use tracing::instrument; @@ -18,7 +18,120 @@ use crate::authz::{ require_observe_auth, }; use crate::odata::extract_tenant; -use crate::state::ServerState; +use crate::state::{ServerState, SpecPublicationGuard}; + +mod list; + +pub use list::{handle_list_wasm_invocations, handle_list_wasm_modules}; + +async fn begin_wasm_generation_mutation( + state: &ServerState, + tenant: &TenantId, +) -> Result { + let guard = state + .begin_spec_publication(tenant) + .await + .map_err(|error| { + ( + StatusCode::SERVICE_UNAVAILABLE, + format!("tenant runtime generation is busy: {error}"), + ) + })?; + Ok(guard) +} + +async fn begin_wasm_generation_read( + state: &ServerState, + tenant: &TenantId, +) -> Result, StatusCode> { + if state.spec_publication_gated(tenant) { + return Err(StatusCode::SERVICE_UNAVAILABLE); + } + let guard = state + .try_begin_tenant_request(tenant) + .await + .ok_or(StatusCode::SERVICE_UNAVAILABLE)?; + if state.spec_publication_gated(tenant) { + return Err(StatusCode::SERVICE_UNAVAILABLE); + } + Ok(guard) +} + +async fn authorize_wasm_mutation( + state: &ServerState, + headers: &HeaderMap, + tenant: &TenantId, + module_name: &str, +) -> Result<(), (StatusCode, String)> { + let mut resource_attrs = BTreeMap::new(); + resource_attrs.insert( + "id".to_string(), + serde_json::Value::String(module_name.to_string()), + ); + resource_attrs.insert( + "module_name".to_string(), + serde_json::Value::String(module_name.to_string()), + ); + if let Some(response) = require_governed_mutation_auth( + state, + headers, + GovernedMutationAuth { + tenant: tenant.as_str(), + action: "manage_wasm", + resource_type: "WasmModule", + resource_id: module_name, + resource_attrs, + module_name: Some(module_name), + from_status: None, + }, + ) + .await + { + return Err(response); + } + Ok(()) +} + +fn known_wasm_tenants(state: &ServerState) -> Result, StatusCode> { + let mut tenants = state + .registry + .read() + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)? + .tenant_ids() + .into_iter() + .map(|tenant| tenant.as_str().to_string()) + .collect::>(); + tenants.extend( + state + .wasm_module_registry + .read() + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)? + .all_modules() + .into_iter() + .map(|(tenant, _, _)| tenant.to_string()), + ); + Ok(tenants) +} + +async fn begin_all_wasm_generation_reads( + state: &ServerState, + tenants: &BTreeSet, +) -> Result>, StatusCode> { + let mut guards = Vec::with_capacity(tenants.len()); + for tenant in tenants { + guards.push(begin_wasm_generation_read(state, &TenantId::new(tenant)).await?); + } + Ok(guards) +} + +fn wasm_authorization_tenant(headers: &HeaderMap) -> String { + headers + .get("x-tenant-id") + .and_then(|value| value.to_str().ok()) + .filter(|tenant| !tenant.is_empty()) + .unwrap_or("system") + .to_string() +} #[derive(Deserialize)] struct WasmModuleUploadJson { @@ -99,32 +212,7 @@ pub async fn handle_upload_wasm_module( body: axum::body::Bytes, ) -> Result, (StatusCode, String)> { let tenant = extract_tenant(&headers, &state)?; - let mut resource_attrs = BTreeMap::new(); - resource_attrs.insert( - "id".to_string(), - serde_json::Value::String(module_name.clone()), - ); - resource_attrs.insert( - "module_name".to_string(), - serde_json::Value::String(module_name.clone()), - ); - if let Some(resp) = require_governed_mutation_auth( - &state, - &headers, - GovernedMutationAuth { - tenant: tenant.as_str(), - action: "manage_wasm", - resource_type: "WasmModule", - resource_id: &module_name, - resource_attrs, - module_name: Some(&module_name), - from_status: None, - }, - ) - .await - { - return Err(resp); - } + authorize_wasm_mutation(&state, &headers, &tenant, &module_name).await?; let module_bytes = decode_wasm_upload_body(&headers, body)?; @@ -153,6 +241,23 @@ pub async fn handle_upload_wasm_module( ) })?; + // Serialize the durable row and live module-name mapping with spec/app + // publication. Compilation above is content-addressed cache warming only; + // no tenant-visible generation changes before this writer is held. + let mut generation_writer = begin_wasm_generation_mutation(&state, &tenant).await?; + authorize_wasm_mutation(&state, &headers, &tenant, &module_name).await?; + let intent = ServerState::spec_publication_intent( + "direct-wasm-upload-v1", + [ + ("module-name", module_name.as_bytes()), + ("sha256", hash.as_bytes()), + ("wasm-bytes", module_bytes.as_ref()), + ], + ); + state + .arm_spec_publication(&mut generation_writer, &tenant, &intent) + .map_err(|error| (StatusCode::CONFLICT, error))?; + // Persist to durable storage first — if durability fails, refuse the upload. // This ensures the module survives restarts before we expose it in memory. // source="upload" so the os-apps install pipeline won't clobber this row at @@ -168,7 +273,6 @@ pub async fn handle_upload_wasm_module( .await { tracing::error!(error = %e, "failed to persist WASM module to durable store"); - state.wasm_engine.evict(&hash); return Err(( StatusCode::INTERNAL_SERVER_ERROR, format!("failed to persist WASM module: {e}"), @@ -177,9 +281,17 @@ pub async fn handle_upload_wasm_module( // Register in module registry after durability is confirmed. { - let mut wasm_reg = state.wasm_module_registry.write().unwrap(); + let mut wasm_reg = state.wasm_module_registry.write().map_err(|error| { + ( + StatusCode::SERVICE_UNAVAILABLE, + format!("WASM registry lock poisoned: {error}"), + ) + })?; wasm_reg.register(&tenant, &module_name, &hash); } + state + .complete_spec_publication_retry(&mut generation_writer, &tenant) + .map_err(|error| (StatusCode::SERVICE_UNAVAILABLE, error))?; let size_bytes = module_bytes.len(); tracing::info!( @@ -237,6 +349,7 @@ pub async fn handle_get_wasm_module_info( Path(module_name): Path, ) -> Result, StatusCode> { let tenant = extract_tenant(&headers, &state).map_err(|(s, _)| s)?; + let _generation = begin_wasm_generation_read(&state, &tenant).await?; let hash = { let wasm_reg = state.wasm_module_registry.read().unwrap(); @@ -269,34 +382,14 @@ pub async fn handle_delete_wasm_module( Path(module_name): Path, ) -> Result, (StatusCode, String)> { let tenant = extract_tenant(&headers, &state)?; - let mut resource_attrs = BTreeMap::new(); - resource_attrs.insert( - "id".to_string(), - serde_json::Value::String(module_name.clone()), - ); - resource_attrs.insert( - "module_name".to_string(), - serde_json::Value::String(module_name.clone()), - ); - if let Some(resp) = require_governed_mutation_auth( - &state, - &headers, - GovernedMutationAuth { - tenant: tenant.as_str(), - action: "manage_wasm", - resource_type: "WasmModule", - resource_id: &module_name, - resource_attrs, - module_name: Some(&module_name), - from_status: None, - }, - ) - .await - { - return Err(resp); - } + authorize_wasm_mutation(&state, &headers, &tenant, &module_name).await?; + + let mut generation_writer = begin_wasm_generation_mutation(&state, &tenant).await?; + authorize_wasm_mutation(&state, &headers, &tenant, &module_name).await?; + let retrying_exact_delete = state.spec_publication_gated(&tenant); - // Get hash before removing from registry (for cache eviction) + // Get hash after joining the tenant-generation writer, before removing it + // from the registry for cache eviction. let hash = { let wasm_reg = state.wasm_module_registry.read().unwrap(); wasm_reg @@ -304,7 +397,7 @@ pub async fn handle_delete_wasm_module( .map(|s| s.to_string()) }; - if hash.is_none() { + if hash.is_none() && !retrying_exact_delete { tracing::warn!("WASM module not found for deletion"); return Err(( StatusCode::NOT_FOUND, @@ -312,6 +405,14 @@ pub async fn handle_delete_wasm_module( )); } + let intent = ServerState::spec_publication_intent( + "direct-wasm-delete-v1", + [("module-name", module_name.as_bytes())], + ); + state + .arm_spec_publication(&mut generation_writer, &tenant, &intent) + .map_err(|error| (StatusCode::CONFLICT, error))?; + // Delete from durable storage first — if durability fails, refuse the delete // so memory stays consistent with the durable store. if let Err(e) = state @@ -327,7 +428,12 @@ pub async fn handle_delete_wasm_module( // Remove from in-memory registry after durability is confirmed. { - let mut wasm_reg = state.wasm_module_registry.write().unwrap(); + let mut wasm_reg = state.wasm_module_registry.write().map_err(|error| { + ( + StatusCode::SERVICE_UNAVAILABLE, + format!("WASM registry lock poisoned: {error}"), + ) + })?; wasm_reg.remove(&tenant, &module_name); } @@ -335,6 +441,9 @@ pub async fn handle_delete_wasm_module( if let Some(ref hash) = hash { state.wasm_engine.evict(hash); } + state + .complete_spec_publication_retry(&mut generation_writer, &tenant) + .map_err(|error| (StatusCode::SERVICE_UNAVAILABLE, error))?; tracing::info!( tenant = %tenant, @@ -347,136 +456,3 @@ pub async fn handle_delete_wasm_module( "module_name": module_name, }))) } - -/// GET /observe/wasm/modules — list all modules (with stats). -/// -/// Admin/System principals see all tenants; others are scoped to `X-Tenant-Id`. -#[instrument(skip_all, fields(otel.name = "GET /observe/wasm/modules"))] -pub async fn handle_list_wasm_modules( - State(state): State, - headers: HeaderMap, -) -> Result, StatusCode> { - require_observe_auth(&state, &headers, "read_wasm", "WasmModule")?; - let tenant_scope = observe_tenant_scope(&state, &headers)?; - - // Collect invocation stats via fan-out across all tenant stores. - let invocation_stats: std::collections::BTreeMap)> = { - let mut stats: std::collections::BTreeMap)> = - std::collections::BTreeMap::new(); - let stores = state.collect_all_metadata_stores().await; - for store in &stores { - if let Ok(rows) = store.load_recent_wasm_invocations(10_000).await { - for row in rows { - let module = row.module_name.clone(); - let success = row.success; - let ts = Some(row.created_at.clone()); - let (total, s_count, last_ts) = stats.entry(module).or_insert((0, 0, None)); - *total += 1; - if success { - *s_count += 1; - } - if ts.is_some() { - *last_ts = ts; - } - } - } - } - stats - }; - - let modules: Vec = { - let wasm_reg = state.wasm_module_registry.read().unwrap(); // ci-ok: infallible lock - - let make_entry = |tenant: &str, name: &str, hash: &str| { - let cached = state.wasm_engine.is_cached(hash); - let (total_invocations, success_count, last_invoked_at) = - invocation_stats.get(name).cloned().unwrap_or((0, 0, None)); - let success_rate = if total_invocations > 0 { - success_count as f64 / total_invocations as f64 - } else { - 0.0 - }; - WasmModuleListEntry { - tenant: tenant.to_string(), - module_name: name.to_string(), - sha256_hash: hash.to_string(), - cached, - total_invocations, - success_count, - success_rate, - last_invoked_at, - } - }; - - let mut entries: Vec = wasm_reg - .all_modules() - .into_iter() - .filter(|(tenant, _, _)| { - tenant_scope - .as_ref() - .is_none_or(|scope| scope.as_str() == *tenant) - }) - .map(|(tenant, name, hash)| make_entry(tenant, name, hash)) - .collect(); - - // Include built-in modules (visible to all tenants, no tenant scope filter). - for (name, hash) in wasm_reg.all_builtins() { - entries.push(make_entry("builtin", name, hash)); - } - - entries - }; - - let total = modules.len(); - Ok(Json(serde_json::json!({ - "modules": modules, - "total": total, - }))) -} - -/// GET /observe/wasm/invocations — query WASM invocation history. -#[instrument(skip_all, fields(otel.name = "GET /observe/wasm/invocations"))] -pub async fn handle_list_wasm_invocations( - State(state): State, - headers: HeaderMap, - Query(params): Query, -) -> Result, StatusCode> { - require_observe_auth(&state, &headers, "read_wasm", "WasmModule")?; - let limit = params.limit.unwrap_or(100).min(10_000); - - let stores = state.collect_all_metadata_stores().await; - let mut all_filtered: Vec = Vec::new(); - for store in &stores { - match store.load_recent_wasm_invocations(limit as i64).await { - Ok(rows) => { - let filtered: Vec = rows - .into_iter() - .filter(|e| { - if let Some(ref mn) = params.module_name - && e.module_name != *mn - { - return false; - } - if let Some(s) = params.success - && e.success != s - { - return false; - } - true - }) - .map(|e| serde_json::to_value(&e).unwrap_or_default()) - .collect(); - all_filtered.extend(filtered); - } - Err(e) => { - tracing::warn!(error = %e, backend = store.backend_name(), "failed to query WASM invocations"); - } - } - } - - let total = all_filtered.len(); - Ok(Json(WasmInvocationResponse { - invocations: all_filtered, - total, - })) -} diff --git a/crates/temper-server/src/observe/wasm/list.rs b/crates/temper-server/src/observe/wasm/list.rs new file mode 100644 index 000000000..11469c941 --- /dev/null +++ b/crates/temper-server/src/observe/wasm/list.rs @@ -0,0 +1,199 @@ +use axum::extract::{Query, State}; + +use super::*; + +/// GET /observe/wasm/modules — list all modules (with stats). +/// +/// Admin/System principals see all tenants; others are scoped to `X-Tenant-Id`. +#[instrument(skip_all, fields(otel.name = "GET /observe/wasm/modules"))] +pub async fn handle_list_wasm_modules( + State(state): State, + headers: HeaderMap, +) -> Result, StatusCode> { + let tenant_scope = observe_tenant_scope(&state, &headers)?; + let stores = state.collect_all_metadata_stores().await; + let mut guarded_tenants = BTreeSet::new(); + let mut _all_generation_guards = Vec::new(); + let _tenant_generation_guard; + if let Some(tenant) = tenant_scope.as_ref() { + _tenant_generation_guard = Some(begin_wasm_generation_read(&state, tenant).await?); + } else { + _tenant_generation_guard = None; + guarded_tenants = known_wasm_tenants(&state)?; + guarded_tenants.insert(wasm_authorization_tenant(&headers)); + for store in &stores { + if let Ok(rows) = store.load_recent_wasm_invocations(10_000).await { + guarded_tenants.extend(rows.into_iter().map(|row| row.tenant)); + } + } + _all_generation_guards = begin_all_wasm_generation_reads(&state, &guarded_tenants).await?; + } + require_observe_auth(&state, &headers, "read_wasm", "WasmModule")?; + + // Collect invocation stats via fan-out across all tenant stores. + let invocation_stats: BTreeMap<(String, String), (usize, usize, Option)> = { + let mut stats = BTreeMap::new(); + for store in &stores { + if let Ok(rows) = store.load_recent_wasm_invocations(10_000).await { + for row in rows { + if tenant_scope + .as_ref() + .is_some_and(|tenant| tenant.as_str() != row.tenant) + { + continue; + } + if tenant_scope.is_none() && !guarded_tenants.contains(&row.tenant) { + return Err(StatusCode::SERVICE_UNAVAILABLE); + } + let module = (row.tenant.clone(), row.module_name.clone()); + let success = row.success; + let ts = Some(row.created_at.clone()); + let (total, s_count, last_ts) = stats.entry(module).or_insert((0, 0, None)); + *total += 1; + if success { + *s_count += 1; + } + if ts.is_some() { + *last_ts = ts; + } + } + } + } + stats + }; + + let modules: Vec = { + let wasm_reg = state.wasm_module_registry.read().unwrap(); // ci-ok: infallible lock + let tenant_modules = wasm_reg.all_modules(); + if tenant_scope.is_none() + && tenant_modules + .iter() + .any(|(tenant, _, _)| !guarded_tenants.contains(*tenant)) + { + return Err(StatusCode::SERVICE_UNAVAILABLE); + } + + let make_entry = |tenant: &str, name: &str, hash: &str| { + let cached = state.wasm_engine.is_cached(hash); + let (total_invocations, success_count, last_invoked_at) = invocation_stats + .get(&(tenant.to_string(), name.to_string())) + .cloned() + .unwrap_or((0, 0, None)); + let success_rate = if total_invocations > 0 { + success_count as f64 / total_invocations as f64 + } else { + 0.0 + }; + WasmModuleListEntry { + tenant: tenant.to_string(), + module_name: name.to_string(), + sha256_hash: hash.to_string(), + cached, + total_invocations, + success_count, + success_rate, + last_invoked_at, + } + }; + + let mut entries: Vec = tenant_modules + .into_iter() + .filter(|(tenant, _, _)| { + tenant_scope + .as_ref() + .is_none_or(|scope| scope.as_str() == *tenant) + }) + .map(|(tenant, name, hash)| make_entry(tenant, name, hash)) + .collect(); + + // Include built-in modules (visible to all tenants, no tenant scope filter). + for (name, hash) in wasm_reg.all_builtins() { + entries.push(make_entry("builtin", name, hash)); + } + + entries + }; + + let total = modules.len(); + Ok(Json(serde_json::json!({ + "modules": modules, + "total": total, + }))) +} + +/// GET /observe/wasm/invocations — query WASM invocation history. +#[instrument(skip_all, fields(otel.name = "GET /observe/wasm/invocations"))] +pub async fn handle_list_wasm_invocations( + State(state): State, + headers: HeaderMap, + Query(params): Query, +) -> Result, StatusCode> { + let tenant_scope = observe_tenant_scope(&state, &headers)?; + let limit = params.limit.unwrap_or(100).min(10_000); + let stores = state.collect_all_metadata_stores().await; + let mut guarded_tenants = BTreeSet::new(); + let mut _all_generation_guards = Vec::new(); + let _tenant_generation_guard; + if let Some(tenant) = tenant_scope.as_ref() { + _tenant_generation_guard = Some(begin_wasm_generation_read(&state, tenant).await?); + } else { + _tenant_generation_guard = None; + guarded_tenants = known_wasm_tenants(&state)?; + guarded_tenants.insert(wasm_authorization_tenant(&headers)); + for store in &stores { + if let Ok(rows) = store.load_recent_wasm_invocations(limit as i64).await { + guarded_tenants.extend(rows.into_iter().map(|row| row.tenant)); + } + } + _all_generation_guards = begin_all_wasm_generation_reads(&state, &guarded_tenants).await?; + } + require_observe_auth(&state, &headers, "read_wasm", "WasmModule")?; + + let mut all_filtered: Vec = Vec::new(); + for store in &stores { + match store.load_recent_wasm_invocations(limit as i64).await { + Ok(rows) => { + if tenant_scope.is_none() + && rows + .iter() + .any(|entry| !guarded_tenants.contains(&entry.tenant)) + { + return Err(StatusCode::SERVICE_UNAVAILABLE); + } + let filtered: Vec = rows + .into_iter() + .filter(|e| { + if tenant_scope + .as_ref() + .is_some_and(|tenant| tenant.as_str() != e.tenant) + { + return false; + } + if let Some(ref mn) = params.module_name + && e.module_name != *mn + { + return false; + } + if let Some(s) = params.success + && e.success != s + { + return false; + } + true + }) + .map(|e| serde_json::to_value(&e).unwrap_or_default()) + .collect(); + all_filtered.extend(filtered); + } + Err(e) => { + tracing::warn!(error = %e, backend = store.backend_name(), "failed to query WASM invocations"); + } + } + } + + let total = all_filtered.len(); + Ok(Json(WasmInvocationResponse { + invocations: all_filtered, + total, + })) +} diff --git a/crates/temper-server/src/odata/bindings.rs b/crates/temper-server/src/odata/bindings.rs index ffe35b806..391ca2410 100644 --- a/crates/temper-server/src/odata/bindings.rs +++ b/crates/temper-server/src/odata/bindings.rs @@ -8,7 +8,7 @@ use temper_runtime::scheduler::sim_now; use temper_runtime::tenant::TenantId; use tracing_opentelemetry::OpenTelemetrySpanExt; -use temper_authz::SecurityContext; +use temper_authz::{PrincipalKind, SecurityContext}; use super::account_verification::enforce_commons_account_verified_for_action; use super::common::run_write_prechecks; @@ -16,15 +16,147 @@ use super::rate_limit::{enforce_commons_write_rate_limit, owner_id_from_action}; use super::response::annotate_entity; use crate::authz::{DenialInput, record_authz_denial, security_context_from_headers}; use crate::blobs::hydrate_blob_refs_for_tenant; +use crate::entity_actor::{ + EntityRecoveryContext, EntityResponse, recover_entity_state_from_stable_sources, +}; +use crate::idempotency::{BoundActionClaim, BoundActionReplayLookup}; use crate::identity::ResolvedIdentity; use crate::request_context::AgentContext; use crate::response::{ODataResponse, odata_error}; -use crate::state::{BoundActionHookContext, DispatchError, DispatchExtOptions, ServerState}; +use crate::state::{ + BoundActionHookContext, DispatchError, DispatchExtOptions, ServerState, TenantGenerationLease, +}; + +mod execute; +mod hook_receipt; + +use execute::{ + BoundActionExecution, execute_bound_action, merge_bound_action_hook_output, + post_action_error_status, run_or_recover_bound_action_hook, +}; fn idempotency_actor_key(tenant: &TenantId, entity_type: &str, entity_id: &str) -> String { format!("{tenant}:{entity_type}:{entity_id}") } +fn canonical_json(value: &serde_json::Value) -> serde_json::Value { + match value { + serde_json::Value::Object(fields) => { + let sorted = fields + .iter() + .map(|(key, value)| (key.clone(), canonical_json(value))) + .collect::>(); + serde_json::Value::Object(sorted.into_iter().collect()) + } + serde_json::Value::Array(values) => { + serde_json::Value::Array(values.iter().map(canonical_json).collect()) + } + scalar => scalar.clone(), + } +} + +fn bound_action_request_fingerprint( + action: &str, + params: &serde_json::Value, + security_ctx: &SecurityContext, +) -> String { + let principal = canonical_json( + &serde_json::to_value(&security_ctx.principal).unwrap_or(serde_json::Value::Null), + ); + let params = canonical_json(params); + let principal = serde_json::to_vec(&principal).unwrap_or_default(); + let params = serde_json::to_vec(¶ms).unwrap_or_default(); + ServerState::spec_publication_intent( + "bound-action-replay", + [ + ("action", action.as_bytes()), + ("params", params.as_slice()), + ("principal", principal.as_slice()), + ], + ) +} + +fn bound_action_operation_fingerprint(action: &str, params: &serde_json::Value) -> String { + let params = canonical_json(params); + let params = serde_json::to_vec(¶ms).unwrap_or_default(); + ServerState::spec_publication_intent( + "bound-action-operation", + [("action", action.as_bytes()), ("params", params.as_slice())], + ) +} + +fn bound_action_durable_idempotency_key( + raw_key: &str, + operation_fingerprint: &str, + request_fingerprint: &str, +) -> (String, String, String) { + let raw_key_digest = ServerState::spec_publication_intent( + "bound-action-idempotency-key", + [("key", raw_key.as_bytes())], + ); + let raw_prefix = format!("temper.bound-action.v2:{raw_key_digest}:"); + let operation_prefix = format!("{raw_prefix}{operation_fingerprint}:"); + let durable_key = format!("{operation_prefix}{request_fingerprint}"); + (raw_prefix, operation_prefix, durable_key) +} + +async fn recover_durable_bound_action_response( + state: &ServerState, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, +) -> Result { + let (store, backend) = state.event_journal().ok_or_else(|| { + "publication replay has no durable event journal; its pinned in-memory proof is required" + .to_string() + })?; + let live_table = state + .registry + .read() + .map_err(|error| format!("registry lock poisoned: {error}"))? + .get_table_live(tenant, entity_type); + let table = if let Some(live_table) = live_table { + live_table + .read() + .map_err(|error| format!("transition table lock poisoned: {error}"))? + .clone() + } else { + state + .transition_tables + .get(entity_type) + .map(|table| (**table).clone()) + .ok_or_else(|| { + format!("No transition table for tenant '{tenant}', entity type '{entity_type}'") + })? + }; + let initial_fields = serde_json::json!({}); + let blob_store = state.blob_store_for_tenant(tenant).ok(); + let recovered = recover_entity_state_from_stable_sources(EntityRecoveryContext { + tenant: tenant.as_str(), + entity_type, + entity_id, + table: &table, + store: &store, + backend, + initial_fields: &initial_fields, + blob_store: blob_store.as_ref(), + }) + .await + .map_err(|error| format!("failed to recover publication replay proof: {error}"))?; + let recovered = recovered.state.ok_or_else(|| { + format!("no durable state exists for publication actor {tenant}:{entity_type}:{entity_id}") + })?; + Ok(EntityResponse { + success: true, + state: recovered, + error: None, + custom_effects: Vec::new(), + scheduled_actions: Vec::new(), + spawn_requests: Vec::new(), + spec_governed: true, + }) +} + #[allow(clippy::too_many_arguments)] pub(super) async fn dispatch_bound_action( state: &ServerState, @@ -39,6 +171,7 @@ pub(super) async fn dispatch_bound_action( await_integration: bool, idempotency_key: Option, resolved_identity: Option<&ResolvedIdentity>, + generation_lease: Option<&TenantGenerationLease>, ) -> axum::response::Response { let http_start = sim_now(); let tracer = opentelemetry::global::tracer("temper"); @@ -101,6 +234,25 @@ pub(super) async fn dispatch_bound_action( }; let mut dispatch_agent_ctx = agent_ctx.clone(); dispatch_agent_ctx.security_ctx = Some(security_ctx.clone()); + let operation_fingerprint = bound_action_operation_fingerprint(action, &body_json); + let request_fingerprint = bound_action_request_fingerprint(action, &body_json, &security_ctx); + let idempotency_key = idempotency_key.filter(|key| !key.trim().is_empty()); + let publication_capable = state + .bound_action_hook + .as_ref() + .is_some_and(|hook| hook.requires_generation_handoff(entity_type, action)); + if publication_capable + && idempotency_key + .as_deref() + .is_none_or(|key| key.trim().is_empty()) + { + return odata_error( + StatusCode::BAD_REQUEST, + "IdempotencyKeyRequired", + "Publication-capable actions require a non-empty Idempotency-Key", + ) + .into_response(); + } // Default-deny: reject actions on entity types with no registered spec. let is_governed = match state.is_entity_type_governed(tenant, entity_type) { @@ -130,130 +282,170 @@ pub(super) async fn dispatch_bound_action( .into_response(); } - let authz_snapshot = match state - .load_authz_resource_snapshot(tenant, entity_type, key_str) - .await - { - Ok(v) => v, - Err(e) => { - http_span.set_status(Status::error(e.clone())); - http_span.set_attribute(OtelKeyValue::new("http.status_code", 500i64)); - let end_time: std::time::SystemTime = sim_now().into(); - http_span.end_with_timestamp(end_time); - let code = if e.contains("registry lock poisoned") { - "RegistryError" - } else { - "ReadError" - }; - return odata_error(StatusCode::INTERNAL_SERVER_ERROR, code, &e).into_response(); + let actor_key = idempotency_actor_key(tenant, entity_type, key_str); + if state.spec_publication_gated(tenant) { + if !publication_capable { + return odata_error( + StatusCode::SERVICE_UNAVAILABLE, + "SpecPublicationInProgress", + "Tenant runtime generation is being published; retry the request", + ) + .into_response(); } - }; - let current_state = authz_snapshot.current_state; - let resource_attrs = authz_snapshot.resource_attrs; - - if let Err(resp) = enforce_commons_account_verified_for_action( - state, - tenant, - entity_type, - ¤t_state.state.fields, - &body_json, - ) - .await - { - http_span.set_status(Status::error("AccountVerificationRequired")); - http_span.set_attribute(OtelKeyValue::new("http.status_code", 403i64)); - let end_time: std::time::SystemTime = sim_now().into(); - http_span.end_with_timestamp(end_time); - return *resp; - } - - if let Err(resp) = enforce_commons_write_rate_limit( - state, - tenant, - entity_type, - owner_id_from_action(¤t_state.state.fields, &body_json), - headers, - agent_ctx, - resolved_identity, - ) - .await - { - http_span.set_status(Status::error("RateLimitExceeded")); - http_span.set_attribute(OtelKeyValue::new("http.status_code", 429i64)); - let end_time: std::time::SystemTime = sim_now().into(); - http_span.end_with_timestamp(end_time); - return resp; - } - - let authz_result = state.authorize_with_context( - &security_ctx, - action, - entity_type, - &resource_attrs, - tenant.as_str(), - ); - if let Err(denial) = authz_result { - let reason = denial.to_string(); - let pd = record_authz_denial( - state, - DenialInput { - tenant: tenant.as_str(), - security_ctx: &security_ctx, - agent_id_override: agent_ctx.agent_id.as_deref(), + let replay = + idempotency_key + .as_deref() + .map_or(BoundActionReplayLookup::Miss, |idempotency_key| { + state.idempotency_cache.lookup_bound_action_replay( + &actor_key, + idempotency_key, + &request_fingerprint, + ) + }); + let (cached, original_params, hook_completed, hook_output) = match replay { + BoundActionReplayLookup::Match { + response, + params, + hook_completed, + hook_output, + } => (*response, params, hook_completed, hook_output), + BoundActionReplayLookup::Pending => { + return odata_error( + StatusCode::SERVICE_UNAVAILABLE, + "IdempotencyInProgress", + "An identical post-action hook is still in progress", + ) + .into_response(); + } + BoundActionReplayLookup::Conflict + if !matches!( + security_ctx.principal.kind, + PrincipalKind::Admin | PrincipalKind::System + ) => + { + return odata_error( + StatusCode::CONFLICT, + "IdempotencyConflict", + "Idempotency-Key was already used by a different action request or principal", + ) + .into_response(); + } + BoundActionReplayLookup::Conflict | BoundActionReplayLookup::Miss => { + let Some(idempotency_key) = idempotency_key.as_deref() else { + return odata_error( + StatusCode::SERVICE_UNAVAILABLE, + "SpecPublicationInProgress", + "Tenant runtime generation is being published; retry the request", + ) + .into_response(); + }; + let (durable_prefix, operation_prefix, durable_key) = + bound_action_durable_idempotency_key( + idempotency_key, + &operation_fingerprint, + &request_fingerprint, + ); + let durable = match recover_durable_bound_action_response( + state, + tenant, + entity_type, + key_str, + ) + .await + { + Ok(durable) => durable, + Err(_) => { + return odata_error( + StatusCode::SERVICE_UNAVAILABLE, + "SpecPublicationInProgress", + "Tenant runtime generation is being published; retry the request", + ) + .into_response(); + } + }; + let processed = &durable.state.processed_idempotency_keys; + let durable_claims = processed + .keys() + .filter(|stored| stored.starts_with(&durable_prefix)) + .collect::>(); + let legacy_conflict = processed.contains_key(idempotency_key); + let exact_match = processed.contains_key(&durable_key); + let privileged_operation_match = matches!( + security_ctx.principal.kind, + PrincipalKind::Admin | PrincipalKind::System + ) && durable_claims.len() == 1 + && durable_claims[0].starts_with(&operation_prefix); + if legacy_conflict + || durable_claims.len() > 1 + || (!durable_claims.is_empty() && !exact_match && !privileged_operation_match) + { + return odata_error( + StatusCode::CONFLICT, + "IdempotencyConflict", + "Idempotency-Key was already durably used by a different or unproved action request", + ) + .into_response(); + } + if !exact_match && !privileged_operation_match { + return odata_error( + StatusCode::SERVICE_UNAVAILABLE, + "SpecPublicationInProgress", + "Tenant runtime generation is being published; retry the exact publication request", + ) + .into_response(); + } + if !state.idempotency_cache.put_bound_action_effects_applied( + &actor_key, + idempotency_key, + durable.clone(), + request_fingerprint.clone(), + body_json.clone(), + ) { + return odata_error( + StatusCode::SERVICE_UNAVAILABLE, + "IdempotencyCapacityExceeded", + "The durable replay could not reserve bounded recovery state; retry later", + ) + .into_response(); + } + (durable, body_json.clone(), false, None) + } + }; + let mut state_json = serde_json::to_value(&cached.state).unwrap_or_default(); + if hook_completed { + merge_bound_action_hook_output(&mut state_json, hook_output.as_ref()); + } else { + let idempotency_key = idempotency_key + .as_deref() + .expect("publication-capable replay requires idempotency key"); + if let Err(error) = run_or_recover_bound_action_hook( + state, + tenant, + entity_type, + key_str, action, - resource_type: entity_type, - resource_id: key_str, - resource_attrs: serde_json::to_value(&resource_attrs).unwrap_or_default(), - reason: &reason, - module_name: None, - from_status: Some(current_state.state.status.clone()), - }, - ) - .await; - - http_span.set_status(Status::error(reason.clone())); - let end_time: std::time::SystemTime = sim_now().into(); - http_span.end_with_timestamp(end_time); - let reason_with_id = format!("{reason} (decision: {})", pd.id); - return odata_error( - StatusCode::FORBIDDEN, - "AuthorizationDenied", - &reason_with_id, - ) - .into_response(); - } - - let current_fields = current_state.state.fields.clone(); - if let Err(resp) = run_write_prechecks( - state, - tenant, - entity_type, - key_str, - action, - "bound_action", - ¤t_fields, - ) - .await - { - http_span.set_status(Status::error("ConstraintViolation")); - http_span.set_attribute(OtelKeyValue::new("http.status_code", 409i64)); - let end_time: std::time::SystemTime = sim_now().into(); - http_span.end_with_timestamp(end_time); - return resp; - } - - // Idempotency cache check - let actor_key = idempotency_actor_key(tenant, entity_type, key_str); - if let Some(ref idem_key) = idempotency_key - && let Some(cached) = state - .idempotency_cache - .get_after_effects_applied(&actor_key, idem_key) - { - let body = annotate_entity( - serde_json::to_value(&cached.state).unwrap_or_default(), - format!("$metadata#{set_name}/$entity"), - None, - ); + &original_params, + &mut state_json, + generation_lease, + &actor_key, + idempotency_key, + &operation_fingerprint, + &request_fingerprint, + ) + .await + { + let status = post_action_error_status(&error); + return odata_error(status, "PostActionHookFailed", &error).into_response(); + } + } + if let Some(idempotency_key) = idempotency_key.as_deref() { + state.idempotency_cache.unpin_bound_action_replay( + &actor_key, + idempotency_key, + &request_fingerprint, + ); + } + let body = annotate_entity(state_json, format!("$metadata#{set_name}/$entity"), None); http_span.set_attribute(OtelKeyValue::new("idempotency.hit", true)); http_span.set_status(Status::Ok); http_span.set_attribute(OtelKeyValue::new("http.status_code", 200i64)); @@ -266,175 +458,31 @@ pub(super) async fn dispatch_bound_action( .into_response(); } - let result = state - .dispatch_tenant_action_ext_typed( + execute_bound_action( + BoundActionExecution { + state, tenant, + set_name, entity_type, key_str, action, - body_json.clone(), - DispatchExtOptions { - agent_ctx: &dispatch_agent_ctx, - await_integration, - await_reactions: true, - }, - ) - .await; - - let http_end: std::time::SystemTime = sim_now().into(); - let response = match result { - Ok(response) => { - if response.success { - // Cache for idempotency - if let Some(ref idem_key) = idempotency_key { - state.idempotency_cache.put_effects_applied( - &actor_key, - idem_key, - response.clone(), - ); - } - - http_span.set_status(Status::Ok); - http_span.set_attribute(OtelKeyValue::new("http.status_code", 200i64)); - - let mut state_json = serde_json::to_value(&response.state).unwrap_or_default(); - if let Some(hook) = state.bound_action_hook.as_ref() { - match hook - .after_bound_action(BoundActionHookContext { - state, - tenant, - entity_type, - entity_id: key_str, - action, - params: &body_json, - state_json: &state_json, - }) - .await - { - Ok(Some(hook_json)) => { - if let (Some(dst), Some(src)) = - (state_json.as_object_mut(), hook_json.as_object()) - { - dst.insert( - "postAction".to_string(), - serde_json::Value::Object(src.clone()), - ); - } - } - Ok(None) => {} - Err(error) => { - http_span.set_status(Status::error(error.clone())); - http_span.set_attribute(OtelKeyValue::new("http.status_code", 500i64)); - return odata_error( - StatusCode::INTERNAL_SERVER_ERROR, - "PostActionHookFailed", - &error, - ) - .into_response(); - } - } - } - hydrate_blob_refs_for_tenant(state, tenant, &mut state_json).await; - let body = - annotate_entity(state_json, format!("$metadata#{set_name}/$entity"), None); - ODataResponse { - status: StatusCode::OK, - body, - } - .into_response() - } else { - http_span.set_status(Status::error(response.error.clone().unwrap_or_default())); - http_span.set_attribute(OtelKeyValue::new("http.status_code", 409i64)); - odata_error( - StatusCode::CONFLICT, - "ActionFailed", - &response.error.unwrap_or_else(|| "Action failed".into()), - ) - .into_response() - } - } - Err(DispatchError::Ungoverned(entity)) => { - let reason = format!( - "Entity type '{entity}' has no registered spec — actions are denied by default" - ); - http_span.set_status(Status::error("EntityTypeNotGoverned")); - http_span.set_attribute(OtelKeyValue::new("http.status_code", 404i64)); - odata_error(StatusCode::NOT_FOUND, "EntityTypeNotGoverned", &reason).into_response() - } - Err(DispatchError::AuthzDenied(reason)) => { - http_span.set_status(Status::error(reason.clone())); - http_span.set_attribute(OtelKeyValue::new("http.status_code", 403i64)); - odata_error(StatusCode::FORBIDDEN, "AuthorizationDenied", &reason).into_response() - } - Err(DispatchError::QuotaExceeded(reason)) => { - http_span.set_status(Status::error(reason.clone())); - http_span.set_attribute(OtelKeyValue::new("http.status_code", 413i64)); - odata_error(StatusCode::PAYLOAD_TOO_LARGE, "StorageCapExceeded", &reason) - .into_response() - } - Err(DispatchError::Conflict(reason)) => { - http_span.set_status(Status::error(reason.clone())); - http_span.set_attribute(OtelKeyValue::new("http.status_code", 409i64)); - odata_error(StatusCode::CONFLICT, "Conflict", &reason).into_response() - } - // ADR-0048: transient exhaustion → 503 Retry-After so clients and - // proxies back off instead of paging someone. - Err(e @ DispatchError::Transient { .. }) => { - let reason = e.to_string(); - http_span.set_status(Status::error(reason.clone())); - http_span.set_attribute(OtelKeyValue::new("http.status_code", 503i64)); - let mut resp = odata_error( - StatusCode::SERVICE_UNAVAILABLE, - "DispatchTransient", - &reason, - ) - .into_response(); - // Retry-After is per-RFC seconds; 1s is a conservative default - // until admission control (ADR-0051) can supply a tuned value. - resp.headers_mut().insert( - axum::http::header::RETRY_AFTER, - axum::http::HeaderValue::from_static("1"), - ); - resp - } - // ADR-0051: admission control declined; caller should back off. - Err(DispatchError::Deferred { retry_after_ms }) => { - let seconds = retry_after_ms.div_ceil(1000).max(1); - let reason = format!("dispatch deferred: retry after {retry_after_ms}ms"); - http_span.set_status(Status::error("DispatchDeferred")); - http_span.set_attribute(OtelKeyValue::new("http.status_code", 503i64)); - let mut resp = - odata_error(StatusCode::SERVICE_UNAVAILABLE, "DispatchDeferred", &reason) - .into_response(); - let value = axum::http::HeaderValue::from_str(&seconds.to_string()) - .unwrap_or_else(|_| axum::http::HeaderValue::from_static("1")); - resp.headers_mut() - .insert(axum::http::header::RETRY_AFTER, value); - resp - } - Err(e) => { - let reason = e.to_string(); - http_span.set_status(Status::error(reason.clone())); - http_span.set_attribute(OtelKeyValue::new("http.status_code", 500i64)); - odata_error(StatusCode::INTERNAL_SERVER_ERROR, "DispatchError", &reason).into_response() - } - }; - - http_span.end_with_timestamp(http_end); - response + body_json, + agent_ctx, + headers, + await_integration, + idempotency_key, + resolved_identity, + generation_lease, + security_ctx, + dispatch_agent_ctx, + operation_fingerprint, + request_fingerprint, + actor_key, + }, + http_span, + ) + .await } #[cfg(test)] -mod tests { - use super::*; - - #[test] - fn idempotency_actor_key_matches_actor_persistence_id_shape() { - let tenant = TenantId::new("acme"); - - assert_eq!( - idempotency_actor_key(&tenant, "WorkCycle", "wc-1"), - "acme:WorkCycle:wc-1" - ); - } -} +mod tests; diff --git a/crates/temper-server/src/odata/bindings/execute.rs b/crates/temper-server/src/odata/bindings/execute.rs new file mode 100644 index 000000000..989e94330 --- /dev/null +++ b/crates/temper-server/src/odata/bindings/execute.rs @@ -0,0 +1,609 @@ +use super::*; + +pub(super) struct BoundActionExecution<'a> { + pub(super) state: &'a ServerState, + pub(super) tenant: &'a TenantId, + pub(super) set_name: &'a str, + pub(super) entity_type: &'a str, + pub(super) key_str: &'a str, + pub(super) action: &'a str, + pub(super) body_json: serde_json::Value, + pub(super) agent_ctx: &'a AgentContext, + pub(super) headers: &'a HeaderMap, + pub(super) await_integration: bool, + pub(super) idempotency_key: Option, + pub(super) resolved_identity: Option<&'a ResolvedIdentity>, + pub(super) generation_lease: Option<&'a TenantGenerationLease>, + pub(super) security_ctx: SecurityContext, + pub(super) dispatch_agent_ctx: AgentContext, + pub(super) operation_fingerprint: String, + pub(super) request_fingerprint: String, + pub(super) actor_key: String, +} + +pub(super) async fn execute_bound_action( + execution: BoundActionExecution<'_>, + mut http_span: S, +) -> axum::response::Response { + let BoundActionExecution { + state, + tenant, + set_name, + entity_type, + key_str, + action, + body_json, + agent_ctx, + headers, + await_integration, + idempotency_key, + resolved_identity, + generation_lease, + security_ctx, + mut dispatch_agent_ctx, + operation_fingerprint, + request_fingerprint, + actor_key, + } = execution; + + let authz_snapshot = match state + .load_authz_resource_snapshot(tenant, entity_type, key_str) + .await + { + Ok(v) => v, + Err(e) => { + http_span.set_status(Status::error(e.clone())); + http_span.set_attribute(OtelKeyValue::new("http.status_code", 500i64)); + let end_time: std::time::SystemTime = sim_now().into(); + http_span.end_with_timestamp(end_time); + let code = if e.contains("registry lock poisoned") { + "RegistryError" + } else { + "ReadError" + }; + return odata_error(StatusCode::INTERNAL_SERVER_ERROR, code, &e).into_response(); + } + }; + let current_state = authz_snapshot.current_state; + let resource_attrs = authz_snapshot.resource_attrs; + + if let Err(resp) = enforce_commons_account_verified_for_action( + state, + tenant, + entity_type, + ¤t_state.state.fields, + &body_json, + ) + .await + { + http_span.set_status(Status::error("AccountVerificationRequired")); + http_span.set_attribute(OtelKeyValue::new("http.status_code", 403i64)); + let end_time: std::time::SystemTime = sim_now().into(); + http_span.end_with_timestamp(end_time); + return *resp; + } + + if let Err(resp) = enforce_commons_write_rate_limit( + state, + tenant, + entity_type, + owner_id_from_action(¤t_state.state.fields, &body_json), + headers, + agent_ctx, + resolved_identity, + ) + .await + { + http_span.set_status(Status::error("RateLimitExceeded")); + http_span.set_attribute(OtelKeyValue::new("http.status_code", 429i64)); + let end_time: std::time::SystemTime = sim_now().into(); + http_span.end_with_timestamp(end_time); + return resp; + } + + let authz_result = state.authorize_with_context( + &security_ctx, + action, + entity_type, + &resource_attrs, + tenant.as_str(), + ); + if let Err(denial) = authz_result { + let reason = denial.to_string(); + let pd = record_authz_denial( + state, + DenialInput { + tenant: tenant.as_str(), + security_ctx: &security_ctx, + agent_id_override: agent_ctx.agent_id.as_deref(), + action, + resource_type: entity_type, + resource_id: key_str, + resource_attrs: serde_json::to_value(&resource_attrs).unwrap_or_default(), + reason: &reason, + module_name: None, + from_status: Some(current_state.state.status.clone()), + }, + ) + .await; + + http_span.set_status(Status::error(reason.clone())); + let end_time: std::time::SystemTime = sim_now().into(); + http_span.end_with_timestamp(end_time); + let reason_with_id = format!("{reason} (decision: {})", pd.id); + return odata_error( + StatusCode::FORBIDDEN, + "AuthorizationDenied", + &reason_with_id, + ) + .into_response(); + } + + let current_fields = current_state.state.fields.clone(); + if let Err(resp) = run_write_prechecks( + state, + tenant, + entity_type, + key_str, + action, + "bound_action", + ¤t_fields, + ) + .await + { + http_span.set_status(Status::error("ConstraintViolation")); + http_span.set_attribute(OtelKeyValue::new("http.status_code", 409i64)); + let end_time: std::time::SystemTime = sim_now().into(); + http_span.end_with_timestamp(end_time); + return resp; + } + + let mut reservation_guard = None; + if let Some(ref idem_key) = idempotency_key { + let (durable_prefix, _operation_prefix, durable_key) = bound_action_durable_idempotency_key( + idem_key, + &operation_fingerprint, + &request_fingerprint, + ); + let durable_conflict = + current_state + .state + .processed_idempotency_keys + .keys() + .any(|stored| { + stored == idem_key + || (stored.starts_with(&durable_prefix) && stored != &durable_key) + }); + if durable_conflict { + return odata_error( + StatusCode::CONFLICT, + "IdempotencyConflict", + "Idempotency-Key was already durably used by a different or unproved action request", + ) + .into_response(); + } + + match state.idempotency_cache.claim_bound_action( + &actor_key, + idem_key, + &request_fingerprint, + &body_json, + ) { + BoundActionClaim::Conflict => { + return odata_error( + StatusCode::CONFLICT, + "IdempotencyConflict", + "Idempotency-Key was already used by a different action request or principal", + ) + .into_response(); + } + BoundActionClaim::AtCapacity => { + return odata_error( + StatusCode::SERVICE_UNAVAILABLE, + "IdempotencyCapacityExceeded", + "This entity has too many idempotent actions in progress; retry later", + ) + .into_response(); + } + BoundActionClaim::Pending => { + return odata_error( + StatusCode::CONFLICT, + "IdempotencyInProgress", + "An identical request with this Idempotency-Key is still in progress", + ) + .into_response(); + } + BoundActionClaim::Match { + response: cached, + params: original_params, + hook_completed, + hook_output, + } => { + let mut state_json = serde_json::to_value(&cached.state).unwrap_or_default(); + if hook_completed { + merge_bound_action_hook_output(&mut state_json, hook_output.as_ref()); + } else { + if let Err(error) = run_or_recover_bound_action_hook( + state, + tenant, + entity_type, + key_str, + action, + &original_params, + &mut state_json, + generation_lease, + &actor_key, + idem_key, + &operation_fingerprint, + &request_fingerprint, + ) + .await + { + let status = post_action_error_status(&error); + http_span.set_status(Status::error(error.clone())); + http_span.set_attribute(OtelKeyValue::new( + "http.status_code", + status.as_u16() as i64, + )); + return odata_error(status, "PostActionHookFailed", &error).into_response(); + } + } + let body = + annotate_entity(state_json, format!("$metadata#{set_name}/$entity"), None); + http_span.set_attribute(OtelKeyValue::new("idempotency.hit", true)); + http_span.set_status(Status::Ok); + http_span.set_attribute(OtelKeyValue::new("http.status_code", 200i64)); + let end_time: std::time::SystemTime = sim_now().into(); + http_span.end_with_timestamp(end_time); + return ODataResponse { + status: StatusCode::OK, + body, + } + .into_response(); + } + BoundActionClaim::Claimed => { + dispatch_agent_ctx.idempotency_key = Some(durable_key); + reservation_guard = Some(state.idempotency_cache.guard_bound_action_reservation( + &actor_key, + idem_key, + &request_fingerprint, + )); + } + } + } + + let result = state + .dispatch_tenant_action_ext_typed( + tenant, + entity_type, + key_str, + action, + body_json.clone(), + DispatchExtOptions { + agent_ctx: &dispatch_agent_ctx, + await_integration, + await_reactions: true, + }, + ) + .await; + + let http_end: std::time::SystemTime = sim_now().into(); + let response = match result { + Ok(response) => { + if response.success { + // Cache for idempotency + if let Some(ref idem_key) = idempotency_key { + if !state.idempotency_cache.put_bound_action_effects_applied( + &actor_key, + idem_key, + response.clone(), + request_fingerprint.clone(), + body_json.clone(), + ) { + return odata_error( + StatusCode::SERVICE_UNAVAILABLE, + "IdempotencyCapacityExceeded", + "The completed action could not reserve bounded replay state; retry later", + ) + .into_response(); + } + if let Some(guard) = reservation_guard.take() { + guard.disarm(); + } + } + + http_span.set_status(Status::Ok); + http_span.set_attribute(OtelKeyValue::new("http.status_code", 200i64)); + + let mut state_json = serde_json::to_value(&response.state).unwrap_or_default(); + let hook_result = if let Some(idem_key) = idempotency_key.as_deref() { + run_or_recover_bound_action_hook( + state, + tenant, + entity_type, + key_str, + action, + &body_json, + &mut state_json, + generation_lease, + &actor_key, + idem_key, + &operation_fingerprint, + &request_fingerprint, + ) + .await + } else { + apply_bound_action_hook( + state, + tenant, + entity_type, + key_str, + action, + &body_json, + &mut state_json, + generation_lease, + &request_fingerprint, + ) + .await + .map(|_| ()) + }; + if let Err(error) = hook_result { + let status = post_action_error_status(&error); + http_span.set_status(Status::error(error.clone())); + http_span.set_attribute(OtelKeyValue::new( + "http.status_code", + status.as_u16() as i64, + )); + return odata_error(status, "PostActionHookFailed", &error).into_response(); + } + hydrate_blob_refs_for_tenant(state, tenant, &mut state_json).await; + let body = + annotate_entity(state_json, format!("$metadata#{set_name}/$entity"), None); + ODataResponse { + status: StatusCode::OK, + body, + } + .into_response() + } else { + http_span.set_status(Status::error(response.error.clone().unwrap_or_default())); + http_span.set_attribute(OtelKeyValue::new("http.status_code", 409i64)); + odata_error( + StatusCode::CONFLICT, + "ActionFailed", + &response.error.unwrap_or_else(|| "Action failed".into()), + ) + .into_response() + } + } + Err(DispatchError::Ungoverned(entity)) => { + let reason = format!( + "Entity type '{entity}' has no registered spec — actions are denied by default" + ); + http_span.set_status(Status::error("EntityTypeNotGoverned")); + http_span.set_attribute(OtelKeyValue::new("http.status_code", 404i64)); + odata_error(StatusCode::NOT_FOUND, "EntityTypeNotGoverned", &reason).into_response() + } + Err(DispatchError::AuthzDenied(reason)) => { + http_span.set_status(Status::error(reason.clone())); + http_span.set_attribute(OtelKeyValue::new("http.status_code", 403i64)); + odata_error(StatusCode::FORBIDDEN, "AuthorizationDenied", &reason).into_response() + } + Err(DispatchError::QuotaExceeded(reason)) => { + http_span.set_status(Status::error(reason.clone())); + http_span.set_attribute(OtelKeyValue::new("http.status_code", 413i64)); + odata_error(StatusCode::PAYLOAD_TOO_LARGE, "StorageCapExceeded", &reason) + .into_response() + } + Err(DispatchError::Conflict(reason)) => { + http_span.set_status(Status::error(reason.clone())); + http_span.set_attribute(OtelKeyValue::new("http.status_code", 409i64)); + odata_error(StatusCode::CONFLICT, "Conflict", &reason).into_response() + } + // ADR-0048: transient exhaustion → 503 Retry-After so clients and + // proxies back off instead of paging someone. + Err(e @ DispatchError::Transient { .. }) => { + let reason = e.to_string(); + http_span.set_status(Status::error(reason.clone())); + http_span.set_attribute(OtelKeyValue::new("http.status_code", 503i64)); + let mut resp = odata_error( + StatusCode::SERVICE_UNAVAILABLE, + "DispatchTransient", + &reason, + ) + .into_response(); + // Retry-After is per-RFC seconds; 1s is a conservative default + // until admission control (ADR-0051) can supply a tuned value. + resp.headers_mut().insert( + axum::http::header::RETRY_AFTER, + axum::http::HeaderValue::from_static("1"), + ); + resp + } + // ADR-0051: admission control declined; caller should back off. + Err(DispatchError::Deferred { retry_after_ms }) => { + let seconds = retry_after_ms.div_ceil(1000).max(1); + let reason = format!("dispatch deferred: retry after {retry_after_ms}ms"); + http_span.set_status(Status::error("DispatchDeferred")); + http_span.set_attribute(OtelKeyValue::new("http.status_code", 503i64)); + let mut resp = + odata_error(StatusCode::SERVICE_UNAVAILABLE, "DispatchDeferred", &reason) + .into_response(); + let value = axum::http::HeaderValue::from_str(&seconds.to_string()) + .unwrap_or_else(|_| axum::http::HeaderValue::from_static("1")); + resp.headers_mut() + .insert(axum::http::header::RETRY_AFTER, value); + resp + } + Err(e) => { + let reason = e.to_string(); + http_span.set_status(Status::error(reason.clone())); + http_span.set_attribute(OtelKeyValue::new("http.status_code", 500i64)); + odata_error(StatusCode::INTERNAL_SERVER_ERROR, "DispatchError", &reason).into_response() + } + }; + + http_span.end_with_timestamp(http_end); + response +} + +#[expect( + clippy::too_many_arguments, + reason = "the durable hook receipt binds every governed request identity component" +)] +pub(super) async fn apply_bound_action_hook( + state: &ServerState, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + action: &str, + params: &serde_json::Value, + state_json: &mut serde_json::Value, + generation_lease: Option<&TenantGenerationLease>, + operation_id: &str, +) -> Result, String> { + let Some(hook) = state.bound_action_hook.as_ref() else { + return Ok(None); + }; + let expected_generation = if hook.requires_generation_handoff(entity_type, action) { + let lease = generation_lease.ok_or_else(|| { + "publication-capable bound action is missing its tenant generation lease".to_string() + })?; + let expected_generation = lease.captured_generation(); + lease.release(); + Some(expected_generation) + } else { + None + }; + let hook_output = hook + .after_bound_action(BoundActionHookContext { + state, + tenant, + entity_type, + entity_id, + action, + params, + state_json, + operation_id, + expected_generation, + }) + .await?; + merge_bound_action_hook_output(state_json, hook_output.as_ref()); + Ok(hook_output) +} + +#[expect( + clippy::too_many_arguments, + reason = "hook recovery verifies the complete durable request and operation identity" +)] +pub(super) async fn run_or_recover_bound_action_hook( + state: &ServerState, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + action: &str, + params: &serde_json::Value, + state_json: &mut serde_json::Value, + generation_lease: Option<&TenantGenerationLease>, + actor_key: &str, + raw_idempotency_key: &str, + operation_fingerprint: &str, + request_fingerprint: &str, +) -> Result<(), String> { + let hook_guard = state.idempotency_cache.guard_bound_action_hook( + actor_key, + raw_idempotency_key, + request_fingerprint, + || state.spec_publication_gated(tenant), + ); + let (_raw_prefix, _operation_prefix, durable_idempotency_key) = + bound_action_durable_idempotency_key( + raw_idempotency_key, + operation_fingerprint, + request_fingerprint, + ); + let receipt = hook_receipt::BoundActionHookReceipt::new( + tenant, + entity_type, + entity_id, + action, + durable_idempotency_key.clone(), + request_fingerprint.to_string(), + false, + None, + ); + if let Some(receipt) = hook_receipt::load_bound_action_hook_receipt(state, &receipt).await? + && receipt.is_completed() + { + let hook_output = receipt.hook_output().cloned(); + merge_bound_action_hook_output(state_json, hook_output.as_ref()); + if !state.idempotency_cache.complete_bound_action_hook( + actor_key, + raw_idempotency_key, + request_fingerprint, + hook_output, + ) { + return Err("completed durable post-action hook could not be cached".to_string()); + } + hook_guard.disarm(); + return Ok(()); + } + hook_receipt::persist_bound_action_hook_receipt(state, &receipt).await?; + + let hook_output = apply_bound_action_hook( + state, + tenant, + entity_type, + entity_id, + action, + params, + state_json, + generation_lease, + &durable_idempotency_key, + ) + .await?; + let receipt = hook_receipt::BoundActionHookReceipt::new( + tenant, + entity_type, + entity_id, + action, + durable_idempotency_key, + request_fingerprint.to_string(), + true, + hook_output.clone(), + ); + hook_receipt::persist_bound_action_hook_receipt(state, &receipt).await?; + if !state.idempotency_cache.complete_bound_action_hook( + actor_key, + raw_idempotency_key, + request_fingerprint, + hook_output, + ) { + return Err("completed post-action hook could not be cached".to_string()); + } + hook_guard.disarm(); + Ok(()) +} + +pub(super) fn merge_bound_action_hook_output( + state_json: &mut serde_json::Value, + hook_output: Option<&serde_json::Value>, +) { + if let Some(hook_json) = hook_output + && let (Some(dst), Some(src)) = (state_json.as_object_mut(), hook_json.as_object()) + { + dst.insert( + "postAction".to_string(), + serde_json::Value::Object(src.clone()), + ); + } +} + +pub(super) fn post_action_error_status(error: &str) -> StatusCode { + if error.contains("runtime generation is busy") + || error.contains("advanced from runtime generation") + { + StatusCode::SERVICE_UNAVAILABLE + } else { + StatusCode::INTERNAL_SERVER_ERROR + } +} diff --git a/crates/temper-server/src/odata/bindings/hook_receipt.rs b/crates/temper-server/src/odata/bindings/hook_receipt.rs new file mode 100644 index 000000000..8b592e415 --- /dev/null +++ b/crates/temper-server/src/odata/bindings/hook_receipt.rs @@ -0,0 +1,248 @@ +use super::*; + +const BOUND_ACTION_HOOK_RECEIPT_EVENT_TYPE: &str = "Temper.Internal.BoundActionHookReceipt.v1"; +const BOUND_ACTION_HOOK_RECEIPT_SCHEMA: &str = "temper.bound-action-hook-receipt.v1"; + +#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize)] +pub(super) struct BoundActionHookReceipt { + schema: String, + tenant: String, + entity_type: String, + entity_id: String, + action: String, + durable_idempotency_key: String, + request_fingerprint: String, + completed: bool, + hook_output: Option, +} + +impl BoundActionHookReceipt { + #[expect( + clippy::too_many_arguments, + reason = "the durable receipt constructor binds every exact governed request component" + )] + pub(super) fn new( + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + action: &str, + durable_idempotency_key: String, + request_fingerprint: String, + completed: bool, + hook_output: Option, + ) -> Self { + Self { + schema: BOUND_ACTION_HOOK_RECEIPT_SCHEMA.to_string(), + tenant: tenant.to_string(), + entity_type: entity_type.to_string(), + entity_id: entity_id.to_string(), + action: action.to_string(), + durable_idempotency_key, + request_fingerprint, + completed, + hook_output, + } + } + + pub(super) fn hook_output(&self) -> Option<&serde_json::Value> { + self.hook_output.as_ref() + } + + pub(super) fn is_completed(&self) -> bool { + self.completed + } + + fn same_claim(&self, other: &Self) -> bool { + self.schema == other.schema + && self.tenant == other.tenant + && self.entity_type == other.entity_type + && self.entity_id == other.entity_id + && self.action == other.action + && self.durable_idempotency_key == other.durable_idempotency_key + && self.request_fingerprint == other.request_fingerprint + } + + fn persistence_id(&self) -> String { + let digest = ServerState::spec_publication_intent( + "bound-action-hook-receipt-id", + [ + ("entity_type", self.entity_type.as_bytes()), + ("entity_id", self.entity_id.as_bytes()), + ( + "durable_idempotency_key", + self.durable_idempotency_key.as_bytes(), + ), + ], + ); + format!("{}:_BoundActionHook:{digest}", self.tenant) + } +} + +async fn load_receipt( + state: &ServerState, + intended: &BoundActionHookReceipt, +) -> Result, String> { + let Some((store, _backend)) = state.event_journal() else { + return Ok(None); + }; + let persistence_id = intended.persistence_id(); + let events = store + .read_events(&persistence_id, 0) + .await + .map_err(|error| format!("failed to read bound-action hook receipt: {error}"))?; + let mut latest = None; + let mut completed = None; + for event in events { + if event.event_type != BOUND_ACTION_HOOK_RECEIPT_EVENT_TYPE { + continue; + } + let receipt = + serde_json::from_value::(event.payload).map_err(|error| { + format!( + "malformed bound-action hook receipt at sequence {}: {error}", + event.sequence_nr + ) + })?; + if !receipt.same_claim(intended) { + return Err( + "durable bound-action hook receipt conflicts with this exact request".to_string(), + ); + } + if receipt.completed { + if let Some(existing) = completed.as_ref() + && existing != &receipt + { + return Err("durable bound-action hook has conflicting completions".to_string()); + } + completed = Some(receipt.clone()); + } else if completed.is_some() { + return Err("durable bound-action hook receipt regressed after completion".to_string()); + } + latest = Some(receipt); + } + let Some(receipt) = completed.or(latest) else { + return Ok(None); + }; + if intended.completed && receipt.completed && receipt.hook_output != intended.hook_output { + return Err("durable bound-action hook has conflicting output".to_string()); + } + Ok(Some(receipt)) +} + +pub(super) async fn load_bound_action_hook_receipt( + state: &ServerState, + intended: &BoundActionHookReceipt, +) -> Result, String> { + load_receipt(state, intended).await +} + +pub(super) async fn persist_bound_action_hook_receipt( + state: &ServerState, + intended: &BoundActionHookReceipt, +) -> Result<(), String> { + let Some((store, _backend)) = state.event_journal() else { + return Ok(()); + }; + if let Some(existing) = load_receipt(state, intended).await? + && (!intended.completed || existing.completed) + { + return Ok(()); + } + + let persistence_id = intended.persistence_id(); + let boundary = store + .journal_boundary(&persistence_id) + .await + .map_err(|error| { + format!("failed to capture bound-action hook receipt boundary: {error}") + })?; + let payload = serde_json::to_value(intended) + .map_err(|error| format!("failed to encode bound-action hook receipt: {error}"))?; + let envelope = temper_runtime::persistence::PersistenceEnvelope { + sequence_nr: 0, + event_type: BOUND_ACTION_HOOK_RECEIPT_EVENT_TYPE.to_string(), + payload, + metadata: temper_runtime::persistence::EventMetadata { + event_id: temper_runtime::scheduler::sim_uuid(), + causation_id: temper_runtime::scheduler::sim_uuid(), + correlation_id: temper_runtime::scheduler::sim_uuid(), + timestamp: temper_runtime::scheduler::sim_now(), + actor_id: persistence_id.clone(), + }, + }; + if let Err(error) = store + .append(&persistence_id, boundary.latest_sequence, &[envelope]) + .await + { + let recovered = load_receipt(state, intended).await?; + let committed = recovered.is_some_and(|receipt| { + receipt.completed == intended.completed + && (!intended.completed || receipt.hook_output == intended.hook_output) + }); + if committed { + return Ok(()); + } + return Err(format!( + "failed to persist bound-action hook receipt: {error}" + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use temper_runtime::ActorSystem; + use temper_runtime::persistence::{EventMetadata, EventStore, PersistenceEnvelope}; + use temper_runtime::scheduler::{sim_now, sim_uuid}; + use temper_store_sim::SimEventStore; + + use super::*; + use crate::registry::SpecRegistry; + use crate::storage::StorageStack; + + #[tokio::test] + async fn malformed_reserved_hook_receipt_fails_closed() { + let store = SimEventStore::no_faults(4_011); + let mut state = ServerState::from_registry( + ActorSystem::new("malformed-hook-receipt"), + SpecRegistry::new(), + ); + state.set_storage_stack(StorageStack::from_sim(store.clone(), None)); + let tenant = TenantId::default(); + let intended = BoundActionHookReceipt::new( + &tenant, + "Order", + "order-1", + "Cancel", + "durable-key".to_string(), + "request-fingerprint".to_string(), + false, + None, + ); + let persistence_id = intended.persistence_id(); + store + .append( + &persistence_id, + 0, + &[PersistenceEnvelope { + sequence_nr: 0, + event_type: BOUND_ACTION_HOOK_RECEIPT_EVENT_TYPE.to_string(), + payload: serde_json::json!({"schema": "malformed-reserved-receipt"}), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: sim_now(), + actor_id: persistence_id.clone(), + }, + }], + ) + .await + .expect("seed malformed reserved receipt"); + + let error = load_bound_action_hook_receipt(&state, &intended) + .await + .expect_err("reserved receipt corruption must block hook replay"); + assert!(error.contains("malformed bound-action hook receipt")); + } +} diff --git a/crates/temper-server/src/odata/bindings/tests/mod.rs b/crates/temper-server/src/odata/bindings/tests/mod.rs new file mode 100644 index 000000000..8398d2919 --- /dev/null +++ b/crates/temper-server/src/odata/bindings/tests/mod.rs @@ -0,0 +1,11 @@ +use super::*; + +#[test] +fn idempotency_actor_key_matches_actor_persistence_id_shape() { + let tenant = TenantId::new("acme"); + + assert_eq!( + idempotency_actor_key(&tenant, "WorkCycle", "wc-1"), + "acme:WorkCycle:wc-1" + ); +} diff --git a/crates/temper-server/src/odata/common.rs b/crates/temper-server/src/odata/common.rs index f62465d1b..80ce13351 100644 --- a/crates/temper-server/src/odata/common.rs +++ b/crates/temper-server/src/odata/common.rs @@ -64,7 +64,7 @@ pub(super) fn has_expand_options(options: &temper_odata::query::types::ExpandOpt /// Resolve an entity set name to an entity type for a tenant. /// /// Tries SpecRegistry first, then legacy entity_set_map. -pub(super) fn resolve_entity_type( +pub(crate) fn resolve_entity_type( state: &ServerState, tenant: &TenantId, entity_set: &str, diff --git a/crates/temper-server/src/odata/mod.rs b/crates/temper-server/src/odata/mod.rs index 7675e97f7..adef1806d 100644 --- a/crates/temper-server/src/odata/mod.rs +++ b/crates/temper-server/src/odata/mod.rs @@ -19,8 +19,7 @@ mod stream_fast_path; mod stream_put; mod write; -#[cfg(feature = "observe")] -pub(crate) use common::extract_tenant; +pub(crate) use common::{extract_tenant, resolve_entity_type}; pub use content_addressed::handle_blob_ingest_raw; pub use read::handle_hints; pub use read::handle_metadata; diff --git a/crates/temper-server/src/odata/nearest.rs b/crates/temper-server/src/odata/nearest.rs index 984d86d5d..d12adab46 100644 --- a/crates/temper-server/src/odata/nearest.rs +++ b/crates/temper-server/src/odata/nearest.rs @@ -23,7 +23,7 @@ use temper_runtime::tenant::TenantId; use super::authz::{LIST_ACTION, READ_ACTION, authorize_read}; use super::common::resolve_entity_type; use super::query_plane_read::QueryPlaneReadBudget; -use super::read_support::materialize_entity_set_entities; +use super::read_support::{CatalogMaterializationPolicy, materialize_entity_set_entities}; use crate::response::odata_error; use crate::state::ServerState; use crate::vector_index::{VectorMetric, parse_vector_property, rank_nearest}; @@ -196,7 +196,7 @@ pub(super) async fn handle_nearest( &entity_type, &set_name, std::slice::from_ref(&reference_id.to_string()), - true, + CatalogMaterializationPolicy::Any, None, ) .await; @@ -355,7 +355,7 @@ pub(super) async fn handle_nearest( &entity_type, &set_name, std::slice::from_ref(&scored.entity_id), - true, + CatalogMaterializationPolicy::Any, None, ) .await; diff --git a/crates/temper-server/src/odata/query_plane_read/keyed.rs b/crates/temper-server/src/odata/query_plane_read/keyed.rs new file mode 100644 index 000000000..24ce122cd --- /dev/null +++ b/crates/temper-server/src/odata/query_plane_read/keyed.rs @@ -0,0 +1,408 @@ +//! Linearizable exact declared-key candidate reads. + +use super::super::authz::{READ_ACTION, authorize_read}; +use super::scan::{ScanCounters, TelemetryInput, base_telemetry, query_options_with_default_page}; +use super::types::{ + QueryPlaneCoverageReport, QueryPlaneFallbackReason, QueryPlaneReadAuthorization, + QueryPlaneReadError, QueryPlaneReadRequest, QueryPlaneReadResult, QueryPlaneReadStrategy, +}; +use crate::blobs::hydrate_blob_refs_for_tenant; +use crate::entity_actor::{EntityRecoveryContext, recover_entity_state_from_stable_sources}; +use crate::odata::read_support::{ + catalog_row_to_entity_body, durable_source_absent_for_catalog_materialization, +}; +use crate::query_eval::apply_query_options; +use crate::storage::{CatalogRowsLoad, load_catalog_rows_by_id}; +use temper_runtime::persistence::EntityKeyLookup; + +const MAX_OWNERSHIP_READ_ATTEMPTS: usize = 3; + +/// How an exact declared-key filter constrains the authoritative read source. +#[derive(Debug, Eq, PartialEq)] +pub(super) enum KeyedCandidateResolution { + /// The query is not an exact declared-key query. + NotApplicable, + /// The declared-key index is not yet complete for the current key-set + /// signature, so neither a hit nor a miss may bound the candidate set. + NeedsAuthoritativeScan, + /// The complete declared-key index supplied a fenced ownership proof. + Authoritative(KeyOwnershipProof), +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct KeyOwnershipProof { + key_name: String, + key_hash: String, + owner: Option, + contract_revision: u64, +} + +impl KeyOwnershipProof { + pub(super) fn owner(&self) -> Option<&str> { + self.owner.as_ref().map(|owner| owner.entity_id.as_str()) + } + + pub(super) fn into_owner(self) -> Option { + self.owner.map(|owner| owner.entity_id) + } +} + +pub(super) enum FencedKeyedRead { + Complete(QueryPlaneReadResult), + NeedsAuthoritativeScan, +} + +async fn materialize_owner_from_durable_sources( + request: &QueryPlaneReadRequest<'_>, + owner: &EntityKeyLookup, + key_name: &str, + key_hash: &str, +) -> Result, QueryPlaneReadError> { + let Some((store, backend)) = request.state.event_journal() else { + return Err(QueryPlaneReadError::KeyOwnershipUnstable); + }; + let table = { + let registry = request + .state + .registry + .read() + .expect("registry lock poisoned"); + registry + .get_table_live(request.tenant, request.entity_type) + .map(|table| table.read().expect("table lock poisoned").clone()) + } + .or_else(|| { + request + .state + .transition_tables + .get(request.entity_type) + .map(|table| (**table).clone()) + }) + .ok_or(QueryPlaneReadError::KeyOwnershipUnstable)?; + let blob_store = request.state.blob_store_for_tenant(request.tenant).ok(); + let source = recover_entity_state_from_stable_sources(EntityRecoveryContext { + tenant: request.tenant.as_str(), + entity_type: request.entity_type, + entity_id: &owner.entity_id, + table: &table, + store: &store, + backend, + initial_fields: &serde_json::json!({}), + blob_store: blob_store.as_ref(), + }) + .await + .map_err(|_| QueryPlaneReadError::KeyOwnershipUnstable)?; + let durable_sequence = source.durable_sequence(); + let Some(recovered) = source.state else { + return materialize_catalog_only_owner(request, owner, key_name, key_hash).await; + }; + + // The key row and journal sequence are co-committed. Requiring equality + // prevents a stale resident actor/body at N-1 from being certified by a + // stable owner row at N, and detects an index that failed to reconcile a + // later journal append. + if durable_sequence != owner.sequence_nr || recovered.status == "Deleted" { + return materialize_catalog_only_owner(request, owner, key_name, key_hash).await; + } + + // Sequence equality is insufficient for snapshot-backed generations: an + // imported snapshot may replace bytes at the same aggregate sequence while + // a stale key row retains that number. Close the ownership proof against the + // recovered fields as well, so a same-sequence source rewrite cannot certify + // the stale indexed owner. + let owns_requested_key = + crate::key_index::derive_entity_key_rows(&table.keys, &recovered.fields, true) + .iter() + .any(|row| row.key_name == key_name && row.key_hash == key_hash); + if !owns_requested_key { + return Ok(None); + } + + let mut body = + serde_json::to_value(recovered).map_err(|_| QueryPlaneReadError::KeyOwnershipUnstable)?; + hydrate_blob_refs_for_tenant(request.state, request.tenant, &mut body).await; + if let Some(object) = body.as_object_mut() { + object.insert( + "@odata.id".to_string(), + serde_json::json!(format!( + "{}('{}')", + request.entity_set_name, owner.entity_id + )), + ); + } + Ok(Some(body)) +} + +/// Materialize an ADR-0077 migration owner whose catalog is durable but whose +/// journal and snapshot are genuinely absent. The exact catalog generation must +/// match the key row, and both durable sources are checked before and after +/// materialization. The surrounding ownership/revision re-read is the final +/// linearization fence. +async fn materialize_catalog_only_owner( + request: &QueryPlaneReadRequest<'_>, + owner: &EntityKeyLookup, + key_name: &str, + key_hash: &str, +) -> Result, QueryPlaneReadError> { + if !durable_source_absent_for_catalog_materialization( + request.state, + request.tenant, + request.entity_type, + &owner.entity_id, + ) + .await + .map_err(|_| QueryPlaneReadError::KeyOwnershipUnstable)? + { + return Ok(None); + } + let Some(query_plane) = request.state.query_plane_store() else { + return Ok(None); + }; + let ids = [owner.entity_id.clone()]; + let rows = load_catalog_rows_by_id( + &query_plane, + request.tenant.as_str(), + request.entity_type, + &ids, + ) + .await + .map_err(|_| QueryPlaneReadError::KeyOwnershipUnstable)?; + let CatalogRowsLoad::Available(mut rows) = rows else { + return Ok(None); + }; + let Some(row) = rows.remove(&owner.entity_id) else { + return Ok(None); + }; + if row.sequence_nr != owner.sequence_nr || row.status == "Deleted" { + return Ok(None); + } + let keys = request + .state + .declared_keys_for(request.tenant, request.entity_type); + let row_owns_requested_key = crate::key_index::derive_entity_key_rows(&keys, &row.fields, true) + .iter() + .any(|row| row.key_name == key_name && row.key_hash == key_hash); + if !row_owns_requested_key { + return Ok(None); + } + let mut body = catalog_row_to_entity_body(request.entity_type, request.entity_set_name, row); + let body_owns_requested_key = body + .get("fields") + .map(|fields| crate::key_index::derive_entity_key_rows(&keys, fields, true)) + .is_some_and(|rows| { + rows.iter() + .any(|row| row.key_name == key_name && row.key_hash == key_hash) + }); + if !body_owns_requested_key { + return Ok(None); + } + hydrate_blob_refs_for_tenant(request.state, request.tenant, &mut body).await; + if !durable_source_absent_for_catalog_materialization( + request.state, + request.tenant, + request.entity_type, + &owner.entity_id, + ) + .await + .map_err(|_| QueryPlaneReadError::KeyOwnershipUnstable)? + { + return Ok(None); + } + Ok(Some(body)) +} + +fn finish_keyed_result( + request: &QueryPlaneReadRequest<'_>, + owner: Option<&EntityKeyLookup>, + body: Option, + authorization: QueryPlaneReadAuthorization, +) -> QueryPlaneReadResult { + let mut authorized = Vec::new(); + if let (Some(owner), Some(body)) = (owner, body) + && (!authorization.enforces_caller() + || authorize_read( + request.state, + request.tenant, + request.security_ctx, + READ_ACTION, + request.entity_type, + &owner.entity_id, + &body, + ) + .is_ok()) + { + authorized.push(body); + } + let options = query_options_with_default_page(request.query_options, request.budget); + let (entities, count) = apply_query_options(authorized, &options); + let returned_count = entities.len(); + let counters = ScanCounters { + candidate_count: usize::from(owner.is_some()), + materialized_count: usize::from(owner.is_some()), + ..ScanCounters::empty() + }; + let telemetry = base_telemetry( + request, + TelemetryInput { + strategy: QueryPlaneReadStrategy::ReadSourceCursor, + fallback_reason: if owner.is_none() { + QueryPlaneFallbackReason::KeyedAbsence + } else { + QueryPlaneFallbackReason::NoFilterPushdown + }, + filter_pushdown: false, + catalog_materialization: false, + coverage: QueryPlaneCoverageReport::default(), + counters, + returned_count, + }, + ); + QueryPlaneReadResult { + entities, + count, + telemetry, + next_skiptoken: None, + } +} + +/// Resolve an exact declared-key filter through the co-committed ownership +/// index only after the current contract has complete coverage. +pub(super) async fn resolve_keyed_candidates( + request: &QueryPlaneReadRequest<'_>, +) -> KeyedCandidateResolution { + let Some(filter) = request.query_options.filter.as_ref() else { + return KeyedCandidateResolution::NotApplicable; + }; + let Some(pairs) = super::super::filter_sql::equality_field_predicates(filter) else { + return KeyedCandidateResolution::NotApplicable; + }; + let keys = request + .state + .declared_keys_for(request.tenant, request.entity_type); + let Some((key_name, key_hash)) = crate::key_index::resolve_query_to_key(&keys, &pairs) else { + return KeyedCandidateResolution::NotApplicable; + }; + let Some((store, _)) = request.state.event_journal() else { + return KeyedCandidateResolution::NeedsAuthoritativeScan; + }; + if !store.supports_authoritative_key_index() { + return KeyedCandidateResolution::NeedsAuthoritativeScan; + } + + let current_key_set = crate::key_index::declared_key_set_signature(&keys); + let revision_before = match store + .key_index_reconciliation_revision(request.tenant.as_str(), request.entity_type) + .await + { + Ok(revision) => revision, + Err(_) => return KeyedCandidateResolution::NeedsAuthoritativeScan, + }; + if !request + .state + .key_index_backfill_complete(request.tenant, request.entity_type, ¤t_key_set) + .await + { + return KeyedCandidateResolution::NeedsAuthoritativeScan; + } + + let lookup = store + .lookup_by_key_with_sequence( + request.tenant.as_str(), + request.entity_type, + &key_name, + &key_hash, + ) + .await; + let revision_after = match store + .key_index_reconciliation_revision(request.tenant.as_str(), request.entity_type) + .await + { + Ok(revision) => revision, + Err(_) => return KeyedCandidateResolution::NeedsAuthoritativeScan, + }; + if revision_before != revision_after { + return KeyedCandidateResolution::NeedsAuthoritativeScan; + } + + match lookup { + Ok(owner) => KeyedCandidateResolution::Authoritative(KeyOwnershipProof { + key_name, + key_hash, + owner, + contract_revision: revision_after, + }), + Err(_) => KeyedCandidateResolution::NeedsAuthoritativeScan, + } +} + +/// Materialize the indexed owner from journal-backed state, then re-read the +/// ownership row. A same-contract transfer changes the row without changing the +/// contract revision; retrying the new owner closes that lookup/materialization +/// gap. A compatible re-read is the linearization point for both hits and misses. +pub(super) async fn read_fenced_keyed_candidate( + request: &QueryPlaneReadRequest<'_>, + proof: KeyOwnershipProof, + authorization: QueryPlaneReadAuthorization, +) -> Result { + let Some((store, _)) = request.state.event_journal() else { + return Ok(FencedKeyedRead::NeedsAuthoritativeScan); + }; + let mut expected_owner = proof.owner; + + for _ in 0..MAX_OWNERSHIP_READ_ATTEMPTS { + let body = match expected_owner.as_ref() { + Some(owner) => { + materialize_owner_from_durable_sources( + request, + owner, + &proof.key_name, + &proof.key_hash, + ) + .await? + } + None => None, + }; + let body_matches_generation = expected_owner.is_none() || body.is_some(); + let result = finish_keyed_result(request, expected_owner.as_ref(), body, authorization); + + let revision_before = match store + .key_index_reconciliation_revision(request.tenant.as_str(), request.entity_type) + .await + { + Ok(revision) if revision == proof.contract_revision => revision, + _ => return Ok(FencedKeyedRead::NeedsAuthoritativeScan), + }; + let observed_owner = match store + .lookup_by_key_with_sequence( + request.tenant.as_str(), + request.entity_type, + &proof.key_name, + &proof.key_hash, + ) + .await + { + Ok(owner) => owner, + Err(_) => return Ok(FencedKeyedRead::NeedsAuthoritativeScan), + }; + let revision_after = match store + .key_index_reconciliation_revision(request.tenant.as_str(), request.entity_type) + .await + { + Ok(revision) => revision, + Err(_) => return Ok(FencedKeyedRead::NeedsAuthoritativeScan), + }; + + if revision_before != revision_after || revision_after != proof.contract_revision { + return Ok(FencedKeyedRead::NeedsAuthoritativeScan); + } + if observed_owner == expected_owner { + if !body_matches_generation { + return Err(QueryPlaneReadError::KeyOwnershipUnstable); + } + return Ok(FencedKeyedRead::Complete(result)); + } + expected_owner = observed_owner; + } + + Err(QueryPlaneReadError::KeyOwnershipUnstable) +} diff --git a/crates/temper-server/src/odata/query_plane_read/mod.rs b/crates/temper-server/src/odata/query_plane_read/mod.rs index 7646bd6e0..1ca36d1f3 100644 --- a/crates/temper-server/src/odata/query_plane_read/mod.rs +++ b/crates/temper-server/src/odata/query_plane_read/mod.rs @@ -1,22 +1,30 @@ //! OData entity-set reads through the query-plane contract. +mod keyed; mod pagination; mod scan; #[cfg(test)] mod tests; mod types; -pub(in crate::odata) use pagination::read_entity_set_from_query_plane; +pub(in crate::odata) use pagination::{ + read_entity_set_for_internal_resolution, read_entity_set_from_query_plane, +}; pub(in crate::odata) use types::{ QueryPlaneReadBudget, QueryPlaneReadError, QueryPlaneReadRequest, QueryPlaneReadResult, }; use super::authz::{LIST_ACTION, authorize_read}; -use super::read_support::missing_catalog_entity_ids; +use super::read_support::{CatalogMaterializationPolicy, missing_catalog_entity_ids}; +use keyed::{ + FencedKeyedRead, KeyedCandidateResolution, read_fenced_keyed_candidate, + resolve_keyed_candidates, +}; use scan::{ ScanCounters, budget_rejection, native_candidate_page_plan, read_from_source_cursor, try_native_page_read, }; +use types::QueryPlaneReadAuthorization; use types::{QueryPlaneCoverageReport, QueryPlaneFallbackReason, QueryPlaneReadStrategy}; fn should_try_native_before_catalog_coverage( @@ -26,67 +34,6 @@ fn should_try_native_before_catalog_coverage( plan.filter_pushdown && request.query_options.count != Some(true) } -/// ADR-0153: when the read's `$filter` is exactly a declared `[[key]]`, resolve -/// it to the single matching `entity_id` via `entity_key_index` — a bounded -/// candidate set (no full-type scan, so the budget cannot trip → no 413). -/// -/// Returns `Some(vec![id])` on a keyed **hit**; `None` otherwise — including a -/// keyed **miss**, which falls back to the full scan because (until the backfill -/// gate lands) a missing key row may be a pre-backfill entity rather than a true -/// absence. So hits are fast and correct now; authoritative absence follows the -/// per-tenant backfill watermark. `$orderby`/`$count` also decline (a point read -/// has neither to honor). -async fn keyed_candidate_ids(request: &QueryPlaneReadRequest<'_>) -> Option> { - if request.query_options.orderby.is_some() || request.query_options.count == Some(true) { - return None; - } - let filter = request.query_options.filter.as_ref()?; - let pairs = super::filter_sql::equality_field_predicates(filter)?; - // Resolve declared keys via the registry-aware path: runtime-installed os-app - // entities (File, Directory, …) live in the per-tenant registry, NOT in - // `transition_tables`. Reading `transition_tables` here would return `None` for - // them, disabling the keyed path so every point read scans and 413s (ARN-68). - let keys = request - .state - .declared_keys_for(request.tenant, request.entity_type); - let (key_name, key_hash) = crate::key_index::resolve_query_to_key(&keys, &pairs)?; - let (store, _) = request.state.event_journal()?; - match store - .lookup_by_key( - request.tenant.as_str(), - request.entity_type, - &key_name, - &key_hash, - ) - .await - { - Ok(Some(entity_id)) => Some(vec![entity_id]), - Ok(None) => { - // A miss is authoritative absence ONLY once the backfill watermark says - // `entity_key_index` is complete for this (tenant, type) — then we can - // answer "not found" with an empty candidate set (no full-type scan, no - // 413). Before the watermark, a missing row may be a pre-backfill entity, - // so we fall back to the scan (correct, just not bounded). This is the - // retirement of #324's reconcile scan, gated per ADR-0153. - // Authoritative absence requires the CURRENT declared key-set to be fully - // backfilled — otherwise a just-declared key (e.g. a newly-added `ws_path`) - // whose rows are not yet assigned would make a present entity read as absent. - let current_key_set = crate::key_index::declared_key_set_signature(&keys); - if request - .state - .key_index_backfill_complete(request.tenant, request.entity_type, ¤t_key_set) - .await - { - Some(Vec::new()) - } else { - None - } - } - // Error: fall back to the full path (never trust a transient failure as absence). - Err(_) => None, - } -} - fn should_reconcile_empty_exact_match_against_authoritative( request: &QueryPlaneReadRequest<'_>, indexed_entity_ids: &[String], @@ -153,21 +100,68 @@ async fn catalog_coverage_report( /// Execute one page of an OData entity-set read through the query-plane /// contract. Pagination (server-driven `@odata.nextLink` continuation) is /// layered on top by [`read_entity_set_from_query_plane`]. +#[cfg(test)] pub(in crate::odata) async fn read_entity_set_page( request: QueryPlaneReadRequest<'_>, ) -> Result { - if let Err(response) = authorize_read( + read_entity_set_page_with_authorization(request, QueryPlaneReadAuthorization::CallerScoped) + .await +} + +async fn read_entity_set_page_with_authorization( + request: QueryPlaneReadRequest<'_>, + authorization: QueryPlaneReadAuthorization, +) -> Result { + if authorization.enforces_caller() + && let Err(response) = authorize_read( + request.state, + request.tenant, + request.security_ctx, + LIST_ACTION, + request.entity_type, + "", + &serde_json::json!({}), + ) + { + return Err(QueryPlaneReadError::AuthorizationDenied(response)); + } + + // Declared-key ownership is co-committed with the journal and is therefore more + // authoritative than the asynchronously maintained field projection. Resolve and + // close a complete-coverage point read before touching unrelated catalog/EAV repair + // debt; its owner body is independently fenced against journal/snapshot state. + let mut keyed = resolve_keyed_candidates(&request).await; + if let KeyedCandidateResolution::Authoritative(proof) = &keyed { + match read_fenced_keyed_candidate(&request, proof.clone(), authorization).await? { + FencedKeyedRead::Complete(result) => return Ok(result), + FencedKeyedRead::NeedsAuthoritativeScan => { + keyed = KeyedCandidateResolution::NeedsAuthoritativeScan; + } + } + } + + // Every remaining plan can trust or fall back through catalog/EAV state, so it + // must first close the type's bounded durable repair ledger. A complete keyed + // hit/miss returned above never observes those projections and cannot be starved + // by unrelated dirty entities. + crate::state::source_fenced_projection::repair_dirty_projections_before_read( request.state, request.tenant, - request.security_ctx, - LIST_ACTION, request.entity_type, - "", - &serde_json::json!({}), - ) { - return Err(QueryPlaneReadError::AuthorizationDenied(response)); - } + request.budget.scan_candidate_budget(), + ) + .await + .map_err(|error| { + tracing::warn!( + tenant = %request.tenant, + entity_type = request.entity_type, + error = %error, + "query projection repair did not reach a stable source generation" + ); + QueryPlaneReadError::ProjectionUnstable + })?; + let keyed_query = !matches!(&keyed, KeyedCandidateResolution::NotApplicable); let native_plan = native_candidate_page_plan(&request); // Set when an empty native page for an exact-match resolution is treated as // a possibly-lagging projection and we fall through to the authoritative @@ -176,7 +170,8 @@ pub(in crate::odata) async fn read_entity_set_page( // turns out to be too large for the reconcile scan, the budget gate bounds // the reconcile to the field-index coverage gap instead of rejecting (ARN-68). let mut reconciling_exact_match_lag = false; - if let Some(plan) = native_plan.clone() + if !keyed_query + && let Some(plan) = native_plan.clone() && should_try_native_before_catalog_coverage(&request, &plan) { let indexed_entity_ids = request @@ -188,7 +183,9 @@ pub(in crate::odata) async fn read_entity_set_page( let (coverage, missing_ids) = catalog_coverage_report(&request, &indexed_entity_ids).await; if coverage.missing == 0 { - if let Some(result) = try_native_page_read(&request, plan, coverage).await { + if let Some(result) = + try_native_page_read(&request, plan, coverage, authorization).await + { let result = result?; if should_reconcile_empty_exact_match_against_authoritative( &request, @@ -212,6 +209,8 @@ pub(in crate::odata) async fn read_entity_set_page( coverage, &missing_ids, QueryPlaneFallbackReason::CatalogCoverageGap, + CatalogMaterializationPolicy::Any, + authorization, ) .await?; return Ok(QueryPlaneReadResult { @@ -221,8 +220,13 @@ pub(in crate::odata) async fn read_entity_set_page( next_skiptoken: None, }); } - } else if let Some(result) = - try_native_page_read(&request, plan, QueryPlaneCoverageReport::default()).await + } else if let Some(result) = try_native_page_read( + &request, + plan, + QueryPlaneCoverageReport::default(), + authorization, + ) + .await { let result = result?; if should_reconcile_empty_exact_match_against_authoritative( @@ -242,23 +246,18 @@ pub(in crate::odata) async fn read_entity_set_page( } } - // ADR-0153 fast path: if the filter is exactly a declared `[[key]]` (the - // shape behind the agents' 413 — `Files?$filter=WorkspaceId eq … and Path eq …`), - // probe `entity_key_index` for the one matching entity_id instead of listing the - // whole entity type. The candidate set becomes bounded (0 or 1), so the rest of - // the read (coverage, budget, materialization, row-auth) runs unchanged and the - // scan budget can never trip. On a miss we fall back to the full list, which still - // covers pre-backfill entities — a safe additive fast path until #324 is retired. - let keyed = keyed_candidate_ids(&request).await; - // A keyed result of `Some([])` is authoritative absence (the watermark is set): - // the co-committed `entity_key_index` has no such entity. We must NOT then run a - // native page read — the eventually-consistent field index could still surface a - // lagging or just-deleted row and contradict the authoritative answer. Fall - // straight through to materialize the empty candidate set. - let keyed_authoritative_absent = matches!(&keyed, Some(ids) if ids.is_empty()); + // A complete key index bounds the candidate set to zero or one. Until the + // current versioned watermark exists, scan all authoritative entity IDs instead: + // a pre-repair row can outlive its deleted owner while the live projection was + // never written, so neither the hit nor the projection is an ownership proof. + let keyed_authoritative_absent = matches!( + &keyed, + KeyedCandidateResolution::Authoritative(proof) if proof.owner().is_none() + ); let all_entity_ids = match keyed { - Some(ids) => ids, - None => { + KeyedCandidateResolution::Authoritative(proof) => proof.into_owner().into_iter().collect(), + KeyedCandidateResolution::NeedsAuthoritativeScan + | KeyedCandidateResolution::NotApplicable => { request .state .list_entity_ids_lazy(request.tenant, request.entity_type) @@ -354,9 +353,13 @@ pub(in crate::odata) async fn read_entity_set_page( }; if let Some(gap_ids) = gap_ids && let Some(plan) = native_plan.clone() - && let Some(page) = - try_native_page_read(&rescue_request, plan, QueryPlaneCoverageReport::default()) - .await + && let Some(page) = try_native_page_read( + &rescue_request, + plan, + QueryPlaneCoverageReport::default(), + authorization, + ) + .await { let page = page?; let gap_len = gap_ids.len(); @@ -379,6 +382,8 @@ pub(in crate::odata) async fn read_entity_set_page( coverage, &missing_ids, QueryPlaneFallbackReason::ProjectionLagReconcile, + CatalogMaterializationPolicy::Any, + authorization, ) .await?; return Ok(QueryPlaneReadResult { @@ -404,11 +409,12 @@ pub(in crate::odata) async fn read_entity_set_page( } let (coverage, missing_ids) = catalog_coverage_report(&request, &all_entity_ids).await; - if !reconciling_exact_match_lag + if !keyed_query + && !reconciling_exact_match_lag && !keyed_authoritative_absent && coverage.missing == 0 && let Some(plan) = native_plan.clone() - && let Some(result) = try_native_page_read(&request, plan, coverage).await + && let Some(result) = try_native_page_read(&request, plan, coverage, authorization).await { return result.map(|result| QueryPlaneReadResult { entities: result.entities, @@ -437,6 +443,12 @@ pub(in crate::odata) async fn read_entity_set_page( coverage, &missing_ids, fallback_reason, + if keyed_query { + CatalogMaterializationPolicy::JournalAbsentOnly + } else { + CatalogMaterializationPolicy::Any + }, + authorization, ) .await?; Ok(QueryPlaneReadResult { diff --git a/crates/temper-server/src/odata/query_plane_read/pagination.rs b/crates/temper-server/src/odata/query_plane_read/pagination.rs index 6e7fa013d..b21c9e6c8 100644 --- a/crates/temper-server/src/odata/query_plane_read/pagination.rs +++ b/crates/temper-server/src/odata/query_plane_read/pagination.rs @@ -27,9 +27,10 @@ use temper_odata::query::types::{ }; use super::scan; +use super::types::QueryPlaneReadAuthorization; use super::{ QueryPlaneReadBudget, QueryPlaneReadError, QueryPlaneReadRequest, QueryPlaneReadResult, - read_entity_set_page, + read_entity_set_page_with_authorization, }; use crate::query_eval::resolve_property; @@ -295,6 +296,26 @@ fn and_filter(base: Option, keyset: FilterExpr) -> FilterExpr { /// only the field shape differs (ARN-97). pub(in crate::odata) async fn read_entity_set_from_query_plane( request: QueryPlaneReadRequest<'_>, +) -> Result { + read_entity_set_with_authorization(request, QueryPlaneReadAuthorization::CallerScoped).await +} + +/// Resolve an entity identity through the same fenced planner without adding +/// collection/row authorization. The enclosing entity GET authorizes the returned +/// body against the caller exactly once. +pub(in crate::odata) async fn read_entity_set_for_internal_resolution( + request: QueryPlaneReadRequest<'_>, +) -> Result { + read_entity_set_with_authorization( + request, + QueryPlaneReadAuthorization::InternalIdentityResolution, + ) + .await +} + +async fn read_entity_set_with_authorization( + request: QueryPlaneReadRequest<'_>, + authorization: QueryPlaneReadAuthorization, ) -> Result { let query_options = request.query_options; let page_size = request.budget.requested_top(query_options); @@ -354,7 +375,7 @@ pub(in crate::odata) async fn read_entity_set_from_query_plane( budget: page_budget, ..request }; - let mut result = read_entity_set_page(page_request).await?; + let mut result = read_entity_set_page_with_authorization(page_request, authorization).await?; // Telemetry is recorded against the original request shape, not the rewritten // page read (which strips `$select` and adds the keyset filter). diff --git a/crates/temper-server/src/odata/query_plane_read/scan/mod.rs b/crates/temper-server/src/odata/query_plane_read/scan/mod.rs index 6defca145..0cc8fa688 100644 --- a/crates/temper-server/src/odata/query_plane_read/scan/mod.rs +++ b/crates/temper-server/src/odata/query_plane_read/scan/mod.rs @@ -7,10 +7,11 @@ pub(super) use read_source::read_from_source_cursor; use temper_odata::query::types::QueryOptions; use super::super::authz::{READ_ACTION, authorize_read, entity_id_from_body}; -use super::super::read_support::materialize_entity_set_entities; +use super::super::read_support::{CatalogMaterializationPolicy, materialize_entity_set_entities}; use super::types::{ - QueryPlaneCoverageReport, QueryPlaneFallbackReason, QueryPlaneReadBudget, QueryPlaneReadError, - QueryPlaneReadRequest, QueryPlaneReadStrategy, QueryPlaneReadTelemetry, + QueryPlaneCoverageReport, QueryPlaneFallbackReason, QueryPlaneReadAuthorization, + QueryPlaneReadBudget, QueryPlaneReadError, QueryPlaneReadRequest, QueryPlaneReadStrategy, + QueryPlaneReadTelemetry, }; use crate::query_eval::apply_query_options; use crate::storage::QueryFieldIndexOrder; @@ -148,18 +149,23 @@ pub(super) fn budget_rejection( } } -fn is_authorized_entity(request: &QueryPlaneReadRequest<'_>, entity: &serde_json::Value) -> bool { +fn is_authorized_entity( + request: &QueryPlaneReadRequest<'_>, + entity: &serde_json::Value, + authorization: QueryPlaneReadAuthorization, +) -> bool { entity_id_from_body(entity).is_some_and(|entity_id| { - authorize_read( - request.state, - request.tenant, - request.security_ctx, - READ_ACTION, - request.entity_type, - entity_id, - entity, - ) - .is_ok() + !authorization.enforces_caller() + || authorize_read( + request.state, + request.tenant, + request.security_ctx, + READ_ACTION, + request.entity_type, + entity_id, + entity, + ) + .is_ok() }) } @@ -174,9 +180,10 @@ pub(super) fn apply_select_only( pub(super) async fn materialize_and_authorize_ids( request: &QueryPlaneReadRequest<'_>, entity_ids: &[String], - prefer_catalog: bool, + catalog_policy: CatalogMaterializationPolicy, counters: &mut ScanCounters, -) -> Vec { + authorization: QueryPlaneReadAuthorization, +) -> Result, QueryPlaneReadError> { counters.candidate_count += entity_ids.len(); let materialized = materialize_entity_set_entities( request.state, @@ -184,26 +191,30 @@ pub(super) async fn materialize_and_authorize_ids( request.entity_type, request.entity_set_name, entity_ids, - prefer_catalog, + catalog_policy, None, ) .await; + if materialized.error.is_some() { + return Err(QueryPlaneReadError::KeyOwnershipUnstable); + } counters.materialized_count += materialized.entities.len(); counters.catalog_shadow_check_budget += materialized.catalog_shadow_check_budget; counters.catalog_shadow_check_scheduled += materialized.catalog_shadow_check_scheduled; - materialized + Ok(materialized .entities .into_iter() - .filter(|entity| is_authorized_entity(request, entity)) - .collect() + .filter(|entity| is_authorized_entity(request, entity, authorization)) + .collect()) } pub(super) async fn materialize_filter_and_authorize_ids( request: &QueryPlaneReadRequest<'_>, entity_ids: &[String], - prefer_catalog: bool, + catalog_policy: CatalogMaterializationPolicy, counters: &mut ScanCounters, -) -> Vec { + authorization: QueryPlaneReadAuthorization, +) -> Result, QueryPlaneReadError> { counters.candidate_count += entity_ids.len(); let materialized = materialize_entity_set_entities( request.state, @@ -211,10 +222,13 @@ pub(super) async fn materialize_filter_and_authorize_ids( request.entity_type, request.entity_set_name, entity_ids, - prefer_catalog, + catalog_policy, None, ) .await; + if materialized.error.is_some() { + return Err(QueryPlaneReadError::KeyOwnershipUnstable); + } counters.materialized_count += materialized.entities.len(); counters.catalog_shadow_check_budget += materialized.catalog_shadow_check_budget; counters.catalog_shadow_check_scheduled += materialized.catalog_shadow_check_scheduled; @@ -229,8 +243,8 @@ pub(super) async fn materialize_filter_and_authorize_ids( materialized.entities }; - entities + Ok(entities .into_iter() - .filter(|entity| is_authorized_entity(request, entity)) - .collect() + .filter(|entity| is_authorized_entity(request, entity, authorization)) + .collect()) } diff --git a/crates/temper-server/src/odata/query_plane_read/scan/native.rs b/crates/temper-server/src/odata/query_plane_read/scan/native.rs index 54ddad29d..a1e950565 100644 --- a/crates/temper-server/src/odata/query_plane_read/scan/native.rs +++ b/crates/temper-server/src/odata/query_plane_read/scan/native.rs @@ -7,8 +7,10 @@ use super::{ base_telemetry, budget_rejection, materialize_filter_and_authorize_ids, }; use crate::odata::query_plane_read::types::{ - QueryPlaneFallbackReason, QueryPlaneReadRequest, QueryPlaneReadStrategy, + QueryPlaneFallbackReason, QueryPlaneReadAuthorization, QueryPlaneReadRequest, + QueryPlaneReadStrategy, }; +use crate::odata::read_support::CatalogMaterializationPolicy; use crate::storage::{QueryFieldIndexOrder, QueryFieldIndexOrderDirection, QueryFieldIndexPage}; fn native_order_by(request: &QueryPlaneReadRequest<'_>) -> Option> { @@ -141,6 +143,7 @@ pub(in crate::odata::query_plane_read) async fn try_native_page_read( request: &QueryPlaneReadRequest<'_>, plan: NativeCandidatePagePlan, coverage: QueryPlaneCoverageReport, + authorization: QueryPlaneReadAuthorization, ) -> Option> { let requested_top = request.budget.requested_top(request.query_options); let skip = request.query_options.skip.unwrap_or(0); @@ -202,9 +205,18 @@ pub(in crate::odata::query_plane_read) async fn try_native_page_read( } let page_len = page.entity_ids.len(); - let authorized = - materialize_filter_and_authorize_ids(request, &page.entity_ids, true, &mut counters) - .await; + let authorized = match materialize_filter_and_authorize_ids( + request, + &page.entity_ids, + CatalogMaterializationPolicy::Any, + &mut counters, + authorization, + ) + .await + { + Ok(authorized) => authorized, + Err(error) => return Some(Err(error)), + }; for entity in authorized { let index = authorized_seen; authorized_seen += 1; diff --git a/crates/temper-server/src/odata/query_plane_read/scan/read_source.rs b/crates/temper-server/src/odata/query_plane_read/scan/read_source.rs index 2d671dcab..d1822cc5c 100644 --- a/crates/temper-server/src/odata/query_plane_read/scan/read_source.rs +++ b/crates/temper-server/src/odata/query_plane_read/scan/read_source.rs @@ -2,14 +2,15 @@ use std::collections::BTreeSet; use super::super::super::authz::entity_id_from_body; use super::super::types::{ - QueryPlaneCoverageReport, QueryPlaneFallbackReason, QueryPlaneReadError, QueryPlaneReadRequest, - QueryPlaneReadStrategy, + QueryPlaneCoverageReport, QueryPlaneFallbackReason, QueryPlaneReadAuthorization, + QueryPlaneReadError, QueryPlaneReadRequest, QueryPlaneReadStrategy, }; use super::{ CandidateScanResult, ScanCounters, TelemetryInput, apply_select_only, base_telemetry, budget_rejection, filter_only_query_options, materialize_and_authorize_ids, query_options_with_default_page, }; +use crate::odata::read_support::CatalogMaterializationPolicy; use crate::query_eval::apply_query_options; async fn read_source_cursor_without_filter_or_count( @@ -17,6 +18,8 @@ async fn read_source_cursor_without_filter_or_count( all_entity_ids: &[String], coverage: QueryPlaneCoverageReport, fallback_reason: QueryPlaneFallbackReason, + catalog_policy: CatalogMaterializationPolicy, + authorization: QueryPlaneReadAuthorization, ) -> Result { let requested_top = request.budget.requested_top(request.query_options); let skip = request.query_options.skip.unwrap_or(0); @@ -28,7 +31,8 @@ async fn read_source_cursor_without_filter_or_count( strategy: QueryPlaneReadStrategy::ReadSourceCursor, fallback_reason, filter_pushdown: false, - catalog_materialization: request.state.query_plane_store().is_some(), + catalog_materialization: catalog_policy.uses_catalog() + && request.state.query_plane_store().is_some(), coverage, counters, returned_count: 0, @@ -55,7 +59,7 @@ async fn read_source_cursor_without_filter_or_count( request, QueryPlaneReadStrategy::ReadSourceCursor, false, - request.state.query_plane_store().is_some(), + catalog_policy.uses_catalog() && request.state.query_plane_store().is_some(), coverage, counters, )); @@ -67,10 +71,11 @@ async fn read_source_cursor_without_filter_or_count( let authorized = materialize_and_authorize_ids( request, &all_entity_ids[offset..end], - request.state.query_plane_store().is_some(), + catalog_policy, &mut counters, + authorization, ) - .await; + .await?; for entity in authorized { let index = authorized_seen; authorized_seen += 1; @@ -93,7 +98,8 @@ async fn read_source_cursor_without_filter_or_count( strategy: QueryPlaneReadStrategy::ReadSourceCursor, fallback_reason, filter_pushdown: false, - catalog_materialization: request.state.query_plane_store().is_some(), + catalog_materialization: catalog_policy.uses_catalog() + && request.state.query_plane_store().is_some(), coverage, counters, returned_count, @@ -112,6 +118,8 @@ async fn read_source_full_proof( mut coverage: QueryPlaneCoverageReport, missing_ids: &[String], fallback_reason: QueryPlaneFallbackReason, + catalog_policy: CatalogMaterializationPolicy, + authorization: QueryPlaneReadAuthorization, ) -> Result { let scan_budget = request.budget.scan_candidate_budget(); if all_entity_ids.len() > scan_budget { @@ -123,7 +131,7 @@ async fn read_source_full_proof( request, QueryPlaneReadStrategy::ReadSourceCursor, false, - request.state.query_plane_store().is_some(), + catalog_policy.uses_catalog() && request.state.query_plane_store().is_some(), coverage, counters, )); @@ -133,10 +141,11 @@ async fn read_source_full_proof( let authorized = materialize_and_authorize_ids( request, all_entity_ids, - request.state.query_plane_store().is_some(), + catalog_policy, &mut counters, + authorization, ) - .await; + .await?; if !missing_ids.is_empty() && request.query_options.filter.is_some() { let missing = missing_ids.iter().cloned().collect::>(); @@ -160,7 +169,8 @@ async fn read_source_full_proof( strategy: QueryPlaneReadStrategy::ReadSourceCursor, fallback_reason, filter_pushdown: false, - catalog_materialization: request.state.query_plane_store().is_some(), + catalog_materialization: catalog_policy.uses_catalog() + && request.state.query_plane_store().is_some(), coverage, counters, returned_count, @@ -179,6 +189,8 @@ pub(in crate::odata::query_plane_read) async fn read_from_source_cursor( coverage: QueryPlaneCoverageReport, missing_ids: &[String], fallback_reason: QueryPlaneFallbackReason, + catalog_policy: CatalogMaterializationPolicy, + authorization: QueryPlaneReadAuthorization, ) -> Result { let needs_full_proof = request.query_options.filter.is_some() || request.query_options.orderby.is_some() @@ -190,6 +202,8 @@ pub(in crate::odata::query_plane_read) async fn read_from_source_cursor( coverage, missing_ids, fallback_reason, + catalog_policy, + authorization, ) .await } else { @@ -198,6 +212,8 @@ pub(in crate::odata::query_plane_read) async fn read_from_source_cursor( all_entity_ids, coverage, fallback_reason, + catalog_policy, + authorization, ) .await } diff --git a/crates/temper-server/src/odata/query_plane_read/tests.rs b/crates/temper-server/src/odata/query_plane_read/tests.rs index 8f2c81b85..3e281edd8 100644 --- a/crates/temper-server/src/odata/query_plane_read/tests.rs +++ b/crates/temper-server/src/odata/query_plane_read/tests.rs @@ -15,7 +15,7 @@ use temper_runtime::ActorSystem; use temper_runtime::scheduler::sim_uuid; use temper_runtime::tenant::TenantId; use temper_spec::csdl::parse_csdl; -use temper_store_turso::TursoEventStore; +use temper_store_turso::{QueryProjectionUpsert, TursoEventStore}; mod dst_projection_lag; mod keyed_existence; @@ -24,6 +24,11 @@ mod proof; const CSDL_XML: &str = include_str!("../../../../../test-fixtures/specs/model.csdl.xml"); const ORDER_IOA: &str = include_str!("../../../../../test-fixtures/specs/order.ioa.toml"); +const ORDER_KEY_SET_SIGNATURE: &str = "v3|7:ws_path[11:WorkspaceId4:Path]"; +const DIRECTORY_KEY_SET_SIGNATURE: &str = "v3|11:name_parent[4:Name11:WorkspaceId8:ParentId]"; +// determinism-ok: test-only admission prevents libSQL local transactions in +// independent proof fixtures from tripping the process-wide native driver. +static TURSO_QUERY_PROOF_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); fn build_order_state(system_name: &str) -> ServerState { let csdl = parse_csdl(CSDL_XML).expect("CSDL should parse"); @@ -85,6 +90,47 @@ async fn upsert_order_projection( .expect("upsert projection"); } +async fn upsert_order_projections( + store: &TursoEventStore, + tenant: &TenantId, + projections: Vec<(String, serde_json::Value, u64)>, +) { + // Keep each statement well below SQLite's parameter/expression budgets while + // avoiding thousands of independent transactions when these proof fixtures + // run concurrently with the rest of the query-plane suite. + for batch in projections.chunks(200) { + let batch = batch + .iter() + .map(|(entity_id, fields, sequence_nr)| { + let mut fields = fields.clone(); + fields["Id"] = serde_json::json!(entity_id); + let state = serde_json::json!({ + "entity_type": "Order", + "entity_id": entity_id, + "status": "Created", + "fields": fields, + "sequence_nr": sequence_nr, + "events": [], + }); + QueryProjectionUpsert { + entity_type: "Order".to_string(), + entity_id: entity_id.clone(), + status: "Created".to_string(), + fields: fields.clone(), + state, + indexed_fields: fields, + sequence_nr: *sequence_nr, + known_new: true, + } + }) + .collect::>(); + store + .upsert_query_projections(tenant.as_str(), &batch) + .await + .expect("batch upsert projections"); + } +} + #[test] fn fallback_reason_labels_are_stable() { assert_eq!(QueryPlaneFallbackReason::None.as_str(), "none"); @@ -249,6 +295,12 @@ async fn row_authorized_count_over_budget_returns_413() { QueryPlaneReadError::InvalidContinuation => { panic!("no $skiptoken was supplied") } + QueryPlaneReadError::KeyOwnershipUnstable => { + panic!("this non-keyed count does not use declared-key ownership") + } + QueryPlaneReadError::ProjectionUnstable => { + panic!("the in-memory query plane has no durable dirty markers") + } } } @@ -290,6 +342,89 @@ async fn row_authorized_first_page_can_stop_after_proof() { ); } +#[tokio::test] +async fn dirty_projection_repair_makes_bounded_progress_before_retrying() { + let db_path = std::env::temp_dir().join(format!( + "temper-query-plane-bounded-dirty-repair-{}.db", + sim_uuid() + )); + let db_url = format!("file:{}", db_path.display()); + let store = TursoEventStore::new(&db_url, None) + .await + .expect("create local turso db"); + let mut state = build_order_state("query-plane-bounded-dirty-repair"); + state.set_storage_stack(StorageStack::from_turso(store.clone())); + create_orders(&state, 11).await; + let tenant = TenantId::default(); + + for index in 0..11 { + store + .remove_query_projection(tenant.as_str(), "Order", &format!("ord-{index:02}")) + .await + .expect("create a durable projection gap"); + } + assert_eq!( + store + .dirty_query_projection_entity_ids(tenant.as_str(), "Order", 20) + .await + .expect("read initial dirty set") + .len(), + 11 + ); + + let security_ctx = SecurityContext::system(); + let query_options = QueryOptions { + filter: Some(FilterExpr::BinaryOp { + left: Box::new(FilterExpr::Property("Id".to_string())), + op: BinaryOperator::Eq, + right: Box::new(FilterExpr::Literal(ODataValue::String( + "ord-10".to_string(), + ))), + }), + top: Some(1), + ..QueryOptions::default() + }; + let request = || QueryPlaneReadRequest { + state: &state, + tenant: &tenant, + security_ctx: &security_ctx, + entity_type: "Order", + entity_set_name: "Orders", + query_options: &query_options, + budget: QueryPlaneReadBudget { + default_page_size: 1, + max_entities: 1, + }, + }; + + assert!(matches!( + read_entity_set_page(request()).await, + Err(QueryPlaneReadError::ProjectionUnstable) + )); + assert_eq!( + store + .dirty_query_projection_entity_ids(tenant.as_str(), "Order", 20) + .await + .expect("read dirty set after bounded repair"), + vec!["ord-10".to_string()], + "a retryable over-budget response must still repair one bounded batch" + ); + + let result = match read_entity_set_page(request()).await { + Ok(result) => result, + Err(_) => panic!("the next read should finish the remaining bounded repair"), + }; + assert_eq!(result.entities.len(), 1); + assert_eq!(result.entities[0]["entity_id"].as_str(), Some("ord-10")); + assert!( + store + .dirty_query_projection_entity_ids(tenant.as_str(), "Order", 1) + .await + .expect("read final dirty set") + .is_empty() + ); +} + #[tokio::test] async fn projection_lagged_actor_write_repairs_missing_catalog_row() { let db_path = @@ -522,19 +657,20 @@ fn id_eq_filter(entity_id: &str) -> FilterExpr { } } -/// ARN-89: an exact-match resolution must surface a just-committed entity even -/// when its asynchronously-projected row has not yet landed. +/// ARN-89: an exact-match resolution must surface a durable entity whose +/// projection was lost before the dirty ledger existed. /// /// Reproduces the NEW case the fix covers: another Order is present in BOTH the /// in-memory index and the turso projection (so catalog `coverage.missing == 0`), /// while the TARGET Order is durable in the journal — reachable only via /// `list_entity_ids_lazy` — and absent from the in-memory index AND the -/// projection. The pre-fix code trusted the empty native page (since coverage +/// projection, with the exact source dirty marker cleared to model that legacy +/// crash gap. The pre-fix code trusted the empty native page (since coverage /// looked complete for the indexed Order) and returned empty. The fix detects /// the pure equality conjunction, distrusts the empty page, and reconciles /// against authoritative state. #[tokio::test] -async fn exact_match_read_is_consistent_when_projection_queued() { +async fn exact_match_read_reconciles_an_untracked_projection_gap() { let db_path = std::env::temp_dir().join(format!("temper-query-plane-exact-lag-{}.db", sim_uuid())); let _ = std::fs::remove_file(&db_path); @@ -577,6 +713,26 @@ async fn exact_match_read_is_consistent_when_projection_queued() { .remove_query_projection(tenant.as_str(), "Order", "ord-target") .await .expect("evict target projection to simulate projection lag"); + let target_persistence_id = format!("{tenant}:Order:ord-target"); + let target_boundary = + temper_runtime::persistence::EventStore::journal_boundary(&store, &target_persistence_id) + .await + .expect("capture target journal boundary"); + assert!( + store + .clear_query_projection_dirty_if_source( + tenant.as_str(), + "Order", + "ord-target", + temper_runtime::persistence::ProjectionSourceFence { + expected_journal_sequence: target_boundary.latest_sequence, + expected_snapshot: None, + }, + ) + .await + .expect("clear the exact dirty generation"), + "the fixture must model a pre-ledger projection gap" + ); // Fresh reader process: empty in-memory index (a server restart). Touch only // `ord-present` so the index holds exactly that one Order, while `ord-target` diff --git a/crates/temper-server/src/odata/query_plane_read/tests/dst_projection_lag.rs b/crates/temper-server/src/odata/query_plane_read/tests/dst_projection_lag.rs index 7fd757e7f..cc4f35c93 100644 --- a/crates/temper-server/src/odata/query_plane_read/tests/dst_projection_lag.rs +++ b/crates/temper-server/src/odata/query_plane_read/tests/dst_projection_lag.rs @@ -23,10 +23,10 @@ use crate::storage::{ QueryProjectionFieldsRow, StorageStack, }; use async_trait::async_trait; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::sync::Mutex; use temper_runtime::persistence::{ - EntityKeyRow, EventMetadata, PersistenceEnvelope, PersistenceError, + EntityKeyRow, EventMetadata, PersistenceEnvelope, PersistenceError, ProjectionSourceFence, }; use temper_runtime::scheduler::{install_deterministic_context, sim_now, sim_uuid}; use temper_store_sim::SimEventStore; @@ -39,9 +39,23 @@ const DST_SEEDS: u64 = 64; /// projection lagging behind the journal — the exact condition that makes the /// real planner fall back to the authoritative scan (and 413 at scale). #[derive(Default)] -struct SimQueryPlane { +pub(super) struct SimQueryPlane { // (entity_type, entity_id) -> catalog row catalog: Mutex>, + dirty: Mutex>, +} + +impl SimQueryPlane { + pub(super) fn mark_dirty(&self, entity_type: &str, entity_id: &str) { + self.dirty + .lock() + .unwrap() + .insert((entity_type.to_string(), entity_id.to_string())); + } + + pub(super) fn dirty_count(&self) -> usize { + self.dirty.lock().unwrap().len() + } } #[async_trait] @@ -69,6 +83,34 @@ impl QueryPlaneStore for SimQueryPlane { Ok(()) } + async fn upsert_projection_if_source( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + status: &str, + fields: &serde_json::Value, + state: &serde_json::Value, + sequence_nr: u64, + _source: ProjectionSourceFence<'_>, + ) -> Result { + self.upsert_projection( + tenant, + entity_type, + entity_id, + status, + fields, + state, + sequence_nr, + ) + .await?; + self.dirty + .lock() + .unwrap() + .remove(&(entity_type.to_string(), entity_id.to_string())); + Ok(true) + } + async fn remove_projection( &self, _tenant: &str, @@ -82,6 +124,60 @@ impl QueryPlaneStore for SimQueryPlane { Ok(()) } + async fn remove_projection_if_source( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + _source: ProjectionSourceFence<'_>, + ) -> Result { + self.remove_projection(tenant, entity_type, entity_id) + .await?; + self.dirty + .lock() + .unwrap() + .remove(&(entity_type.to_string(), entity_id.to_string())); + Ok(true) + } + + async fn clear_projection_dirty_if_source( + &self, + _tenant: &str, + entity_type: &str, + entity_id: &str, + _source: ProjectionSourceFence<'_>, + ) -> Result { + self.dirty + .lock() + .unwrap() + .remove(&(entity_type.to_string(), entity_id.to_string())); + Ok(true) + } + + async fn remove_projection_if_exact( + &self, + _tenant: &str, + entity_type: &str, + entity_id: &str, + status: &str, + fields: &serde_json::Value, + state: &serde_json::Value, + sequence_nr: u64, + ) -> Result { + let key = (entity_type.to_string(), entity_id.to_string()); + let mut catalog = self.catalog.lock().unwrap(); + let exact = catalog.get(&key).is_some_and(|row| { + row.status == status + && &row.fields == fields + && row.state.as_ref() == Some(state) + && row.sequence_nr == sequence_nr + }); + if exact { + catalog.remove(&key); + } + Ok(exact) + } + async fn query_field_index( &self, _tenant: &str, @@ -138,6 +234,24 @@ impl QueryPlaneStore for SimQueryPlane { ) -> Result>, PersistenceError> { Ok(None) } + + async fn dirty_projection_entity_ids( + &self, + _tenant: &str, + entity_type: &str, + limit: usize, + ) -> Result>, PersistenceError> { + Ok(Some( + self.dirty + .lock() + .unwrap() + .iter() + .filter(|(dirty_type, _)| dirty_type == entity_type) + .take(limit) + .map(|(_, entity_id)| entity_id.clone()) + .collect(), + )) + } } fn envelope(event_type: &str, payload: serde_json::Value) -> PersistenceEnvelope { @@ -155,6 +269,21 @@ fn envelope(event_type: &str, payload: serde_json::Value) -> PersistenceEnvelope } } +fn create_envelope(params: serde_json::Value) -> PersistenceEnvelope { + let event = crate::entity_actor::EntityEvent { + action: "Create".to_string(), + from_status: "Draft".to_string(), + to_status: "Draft".to_string(), + timestamp: sim_now(), + params, + idempotency_key: None, + }; + envelope( + "Create", + serde_json::to_value(event).expect("serialize create event"), + ) +} + fn doc_key_hash(workspace: &str, path: &str) -> String { crate::key_index::canonical_key_hash( "ws_path", @@ -238,7 +367,7 @@ async fn dst_projection_lag_413_eliminated_by_keyed_index() { for i in 0..16usize { let pid = format!("{tenant}:Order:noise-{seed}-{i:02}"); events - .append(&pid, 0, &[envelope("Create", serde_json::json!({}))]) + .append(&pid, 0, &[create_envelope(serde_json::json!({}))]) .await .unwrap(); } @@ -251,8 +380,7 @@ async fn dst_projection_lag_413_eliminated_by_keyed_index() { .append_with_keys( &target_pid, 0, - &[envelope( - "Create", + &[create_envelope( serde_json::json!({ "WorkspaceId": ws, "Path": target_path }), )], &[EntityKeyRow { @@ -330,9 +458,12 @@ async fn dst_projection_lag_413_eliminated_by_keyed_index() { "seed {seed}: same keyed filter shape with NO key row must still 413 (pre-backfill scan fallback)" ); - // GREEN (the fix): the SAME workspace + budget + lag, but the $filter is - // exactly the declared key and the co-committed key row exists. The keyed - // fast path bounds the candidate set to the single id -> no scan -> 200. + // GREEN (the fix): certify that the complete current repair covers this key + // declaration. The SAME workspace + budget + lag can now trust the + // co-committed hit and bound the candidate set to one id -> no scan -> 200. + state + .mark_key_index_backfilled(&tenant, "Order", ORDER_KEY_SET_SIGNATURE) + .await; let keyed = QueryOptions { filter: Some(eq_filter(ws, target_path)), ..QueryOptions::default() @@ -358,3 +489,254 @@ async fn dst_projection_lag_413_eliminated_by_keyed_index() { ); } } + +/// ARN-238 restart shape: durable enumeration contains both a tombstoned former +/// owner and the live replacement, while the query projection contains only the +/// replacement. A declared-key filter must never materialize the tombstone from the +/// coverage gap. Both the ordinary and `$count=true` point lookups must prefer the +/// co-committed key index over the lagging native page, so the stale projection can +/// neither widen the result nor inflate the count. +#[tokio::test] +async fn dst_tombstone_never_resolves_declared_key_after_restart() { + for seed in 0..DST_SEEDS { + let (_guard, _clock, _id) = install_deterministic_context(seed); + let qp = std::sync::Arc::new(SimQueryPlane::default()); + let (state, _events) = sim_state(seed, qp.clone()); + let tenant = TenantId::default(); + let former_id = format!("former-{seed}"); + let fields = serde_json::json!({ + "Id": former_id.clone(), + "WorkspaceId": "ws-reclaim", + "Path": "/same-key" + }); + + state + .get_or_create_tenant_entity(&tenant, "Order", &former_id, fields.clone()) + .await + .expect("create former owner"); + state + .delete_tenant_entity(&tenant, "Order", &former_id) + .await + .expect("delete former owner"); + let replacement_id = format!("replacement-{seed}"); + state + .get_or_create_tenant_entity( + &tenant, + "Order", + &replacement_id, + serde_json::json!({ + "Id": replacement_id.clone(), + "WorkspaceId": "ws-reclaim", + "Path": "/same-key" + }), + ) + .await + .expect("replacement reclaims key"); + + // Simulate restart hydration: durable enumeration sees both streams. The + // query plane still holds the former owner's pre-delete live row — the exact + // lag shape produced when projection removal is delayed or crash-lost. + state.populate_index_from_store(&tenant).await; + state + .mark_key_index_backfilled(&tenant, "Order", ORDER_KEY_SET_SIGNATURE) + .await; + let stale_state = serde_json::json!({ + "entity_type": "Order", + "entity_id": former_id.clone(), + "status": "Draft", + "fields": fields, + "sequence_nr": 1, + }); + qp.upsert_projection( + tenant.as_str(), + "Order", + &former_id, + "Draft", + &stale_state["fields"], + &stale_state, + 1, + ) + .await + .expect("seed lagging pre-delete catalog row"); + let security_ctx = SecurityContext::system(); + let budget = QueryPlaneReadBudget { + default_page_size: 10, + max_entities: 10, + }; + for (include_count, include_orderby) in [(false, false), (true, false), (false, true)] { + let options = QueryOptions { + filter: Some(eq_filter("ws-reclaim", "/same-key")), + count: include_count.then_some(true), + orderby: include_orderby.then(|| { + vec![OrderByClause { + property: "Path".to_string(), + direction: OrderDirection::Desc, + }] + }), + ..QueryOptions::default() + }; + let result = read_entity_set_page(QueryPlaneReadRequest { + state: &state, + tenant: &tenant, + security_ctx: &security_ctx, + entity_type: "Order", + entity_set_name: "Orders", + query_options: &options, + budget, + }) + .await; + let result = match result { + Ok(result) => result, + Err(_) => panic!("declared-key read remains bounded and visible"), + }; + assert_eq!( + result.entities.len(), + 1, + "seed {seed}, count={include_count}, orderby={include_orderby}" + ); + assert_eq!( + result.entities[0]["entity_id"], replacement_id, + "seed {seed}: tombstoned former owner must never resolve" + ); + if include_count { + assert_eq!(result.count, Some(1)); + } else { + assert!(result.telemetry.candidate_count <= 1); + } + } + } +} + +/// Rollout/migration shape: an entity was keyed before ADR-0192, its delete journal +/// event committed without exact key reconciliation, and projection removal was +/// crash-lost. Until the v3 repair reaches this stream, neither the stale key hit nor +/// the pre-delete live catalog row may expose the durable tombstone. +#[tokio::test] +async fn dst_pre_v2_stale_key_hit_never_returns_crash_lost_live_projection() { + for seed in 0..DST_SEEDS { + let (_guard, _clock, _id) = install_deterministic_context(seed); + let qp = std::sync::Arc::new(SimQueryPlane::default()); + let (state, events) = sim_state(seed, qp.clone()); + let tenant = TenantId::default(); + let entity_id = format!("legacy-deleted-{seed}"); + let fields = serde_json::json!({ + "Id": entity_id.clone(), + "WorkspaceId": "ws-legacy", + "Path": "/stale-key" + }); + + state + .get_or_create_tenant_entity(&tenant, "Order", &entity_id, fields.clone()) + .await + .expect("create pre-v3 owner"); + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let current_sequence = events + .read_events(&persistence_id, 0) + .await + .expect("read pre-delete history") + .last() + .expect("create event exists") + .sequence_nr; + let timestamp = sim_now(); + let tombstone = crate::entity_actor::EntityEvent { + action: "Deleted".to_string(), + from_status: "Draft".to_string(), + to_status: "Deleted".to_string(), + timestamp, + params: serde_json::json!({}), + idempotency_key: None, + }; + // Legacy event-only append: durable delete advances, stale key row remains. + events + .append( + &persistence_id, + current_sequence, + &[envelope( + "Deleted", + serde_json::to_value(tombstone).expect("serialize tombstone"), + )], + ) + .await + .expect("append legacy tombstone"); + assert_eq!( + events + .lookup_by_key( + tenant.as_str(), + "Order", + "ws_path", + &doc_key_hash("ws-legacy", "/stale-key"), + ) + .await + .expect("lookup stale legacy key"), + Some(entity_id.clone()), + "precondition: legacy delete left stale ownership" + ); + + // Restart the actor view, but retain the crash-lost pre-delete catalog row. + state.stop_and_remove_entity(&tenant, "Order", &entity_id); + let stale_state = serde_json::json!({ + "entity_type": "Order", + "entity_id": entity_id.clone(), + "status": "Draft", + "fields": fields, + "sequence_nr": current_sequence, + }); + qp.upsert_projection( + tenant.as_str(), + "Order", + &entity_id, + "Draft", + &stale_state["fields"], + &stale_state, + current_sequence, + ) + .await + .expect("seed crash-lost live projection"); + assert!( + !state + .key_index_backfill_complete(&tenant, "Order", ORDER_KEY_SET_SIGNATURE) + .await, + "precondition: v3 repair has not certified this type" + ); + + let security_ctx = SecurityContext::system(); + let budget = QueryPlaneReadBudget { + default_page_size: 10, + max_entities: 10, + }; + for (include_count, include_orderby) in [(false, false), (true, false), (false, true)] { + let options = QueryOptions { + filter: Some(eq_filter("ws-legacy", "/stale-key")), + count: include_count.then_some(true), + orderby: include_orderby.then(|| { + vec![OrderByClause { + property: "Path".to_string(), + direction: OrderDirection::Asc, + }] + }), + ..QueryOptions::default() + }; + let result = read_entity_set_page(QueryPlaneReadRequest { + state: &state, + tenant: &tenant, + security_ctx: &security_ctx, + entity_type: "Order", + entity_set_name: "Orders", + query_options: &options, + budget, + }) + .await; + let result = match result { + Ok(result) => result, + Err(_) => panic!("migration-state read stays within the small proof budget"), + }; + assert!( + result.entities.is_empty(), + "seed {seed}, count={include_count}, orderby={include_orderby}: durable tombstone must win over stale key/catalog" + ); + if include_count { + assert_eq!(result.count, Some(0)); + } + } + } +} diff --git a/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence.rs b/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence.rs index 90073d3a0..1217198a5 100644 --- a/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence.rs +++ b/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence.rs @@ -6,9 +6,19 @@ //! `directory_root_souls_scenario_on_postgres` (gated on `DATABASE_URL`). use super::*; -use temper_runtime::persistence::EventStore; +use temper_runtime::persistence::{ + EventMetadata, EventStore, KeyIndexBackfillFence, PersistenceEnvelope, SnapshotBackfillFence, +}; use temper_store_sim::{SimEventStore, SimFaultConfig}; +use super::dst_projection_lag::SimQueryPlane; +use crate::storage::QueryPlaneStore; + +mod backend_authority; +mod backfill_sources; +mod contract_race; +mod ownership_race; + #[test] fn declared_keys_resolve_from_registry_not_just_transition_tables() { // ARN-68 root cause: runtime-installed os-app entities (File, Directory, …) @@ -54,6 +64,21 @@ fn build_order_state_with_sim(system_name: &str) -> (ServerState, SimEventStore) (state, store) } +async fn current_sequence( + store: &SimEventStore, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, +) -> u64 { + store + .read_events(&format!("{tenant}:{entity_type}:{entity_id}"), 0) + .await + .expect("journal is readable") + .last() + .map(|event| event.sequence_nr) + .unwrap_or(0) +} + const DIRECTORY_IOA: &str = include_str!("../../../../../../test-fixtures/specs/directory.ioa.toml"); @@ -157,15 +182,52 @@ async fn seed_dir( .expect("create directory"); } +/// Journal a Directory through the legacy non-key-co-committing path. This models +/// durable duplicates written before declared-key enforcement existed; current actor +/// writes correctly reject the same duplicate and therefore cannot seed this repair +/// scenario themselves. +fn legacy_directory_create( + name: &str, + path: &str, + workspace: &str, + parent: Option<&str>, +) -> PersistenceEnvelope { + let mut payload = serde_json::Map::new(); + payload.insert("Name".to_string(), serde_json::json!(name)); + payload.insert("Path".to_string(), serde_json::json!(path)); + payload.insert("WorkspaceId".to_string(), serde_json::json!(workspace)); + if let Some(parent) = parent { + payload.insert("ParentId".to_string(), serde_json::json!(parent)); + } + let timestamp = temper_runtime::scheduler::sim_now(); + PersistenceEnvelope { + sequence_nr: 0, + event_type: "Create".to_string(), + payload: serde_json::json!({ + "action": "Create", + "from_status": "Active", + "to_status": "Active", + "timestamp": timestamp, + "params": serde_json::Value::Object(payload), + }), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: "legacy-directory-writer".to_string(), + }, + } +} + /// THE souls-scenario proof, with the REAL Directory spec/key and the duplicate-root /// case that misled the prod read. End-to-end through the real read path: /// 1. > budget directories incl. a root (absent ParentId) + DUPLICATE roots (same /// key) + subdirs → a non-keyed root lookup scans > budget and 413s (the bug). -/// 2. The backfill keys the root (absent ParentId → null), correctly skips the -/// duplicate roots as key-conflicts (not failures), and watermarks Directory. -/// 3. Post-backfill: a NEW workspace's root lookup resolves to authoritative-absent -/// (empty, NO 413) — so `ensure_dirs` creates the root and the soul write -/// proceeds; and the existing workspace's root lookup resolves (keyed hit). +/// 2. The backfill keys one root (absent ParentId → null), surfaces the historical +/// duplicates as conflicts, and withholds the authoritative watermark. +/// 3. Both a new workspace miss and an existing stale/incomplete hit remain +/// scan-safe (and over budget) until operators repair the invalid duplicates. #[tokio::test] async fn directory_root_lookup_souls_scenario_with_real_key_and_duplicates() { let (state, store) = build_dir_state_with_sim("dir-souls"); @@ -176,8 +238,16 @@ async fn directory_root_lookup_souls_scenario_with_real_key_and_duplicates() { // Workspace wsA: a real root (no ParentId), TWO duplicate roots (same key), and // enough subdirs that the total Directory count exceeds the scan budget below. seed_dir(&state, &agent_ctx, "wsA-root", "/", "/", "wsA", None).await; - seed_dir(&state, &agent_ctx, "wsA-root-dup1", "/", "/", "wsA", None).await; - seed_dir(&state, &agent_ctx, "wsA-root-dup2", "/", "/", "wsA", None).await; + for duplicate in ["wsA-root-dup1", "wsA-root-dup2"] { + store + .append( + &format!("{tenant}:Directory:{duplicate}"), + 0, + &[legacy_directory_create("/", "/", "wsA", None)], + ) + .await + .expect("seed legacy duplicate root"); + } for i in 0..12 { seed_dir( &state, @@ -222,16 +292,16 @@ async fn directory_root_lookup_souls_scenario_with_real_key_and_duplicates() { Err(_) => panic!("expected QueryTooLarge before watermark, got another error"), } - // (2) Backfill: clear the lazy index (fresh-boot), run it, confirm the real root - // is keyed, the duplicates collided (still watermarked), and the type is watermarked. + // (2) Backfill: clear the lazy index (fresh-boot), run it, confirm one real root + // is keyed but the duplicate conflict prevents a false-complete watermark. state.entity_index.write().unwrap().clear(); state.entity_index_hydrated.write().unwrap().clear(); state.populate_key_index_from_snapshots(&tenant).await; assert!( - state - .key_index_backfill_complete(&tenant, "Directory", "name_parent") + !state + .key_index_backfill_complete(&tenant, "Directory", DIRECTORY_KEY_SET_SIGNATURE) .await, - "Directory must watermark — duplicate roots are key-conflict skips, not failures" + "historical duplicate claims must block authoritative absence" ); assert!( store @@ -247,9 +317,9 @@ async fn directory_root_lookup_souls_scenario_with_real_key_and_duplicates() { "the wsA root (absent ParentId) must be keyed" ); - // (3a) The souls case: a NEW workspace (no root) → authoritative-absent → empty, - // NO 413. This is what lets ensure_dirs create the root and the soul write proceed. - let r = match read_entity_set_page(QueryPlaneReadRequest { + // (3a) A NEW workspace miss remains scan-safe. This small Sim query is over the + // configured proof budget, so it rejects instead of claiming false absence. + match read_entity_set_page(QueryPlaneReadRequest { state: &state, tenant: &tenant, security_ctx: &security_ctx, @@ -260,21 +330,16 @@ async fn directory_root_lookup_souls_scenario_with_real_key_and_duplicates() { }) .await { - Ok(r) => r, - Err(_) => panic!("no 413 once watermarked — keyed miss must be authoritative absence"), - }; - assert!( - r.entities.is_empty(), - "a workspace with no root resolves to absent" - ); - assert_eq!( - r.telemetry.fallback_reason, - QueryPlaneFallbackReason::KeyedAbsence - ); + Err(QueryPlaneReadError::QueryTooLarge { .. }) => {} + Ok(_) => panic!("duplicate-conflicted type must not claim authoritative absence"), + Err(_) => panic!("expected scan-safe QueryTooLarge after conflict"), + } - // (3b) An existing workspace's root lookup resolves (hits the one keyed root, - // despite the duplicates) — no 413, returns exactly one root. - let r = match read_entity_set_page(QueryPlaneReadRequest { + // (3b) An existing row is not enough to prove ownership while duplicate + // conflicts withhold the v3 watermark. The hit must take the same + // authoritative scan and reject honestly rather than trust a potentially stale + // pre-v3 claim. + match read_entity_set_page(QueryPlaneReadRequest { state: &state, tenant: &tenant, security_ctx: &security_ctx, @@ -285,14 +350,10 @@ async fn directory_root_lookup_souls_scenario_with_real_key_and_duplicates() { }) .await { - Ok(r) => r, - Err(_) => panic!("existing root must resolve without 413"), - }; - assert_eq!( - r.entities.len(), - 1, - "wsA root lookup resolves to the keyed root" - ); + Err(QueryPlaneReadError::QueryTooLarge { .. }) => {} + Ok(_) => panic!("an incomplete keyed hit must not bypass the proof budget"), + Err(_) => panic!("expected scan-safe QueryTooLarge for incomplete hit"), + } } /// Seed a Directory on a Postgres-backed state under an explicit tenant (the PG souls @@ -329,9 +390,9 @@ async fn pg_seed_dir( /// event store AND the query-plane), gated on `DATABASE_URL`; unique tenant for /// isolation. Same shape as the sim proof, but against the actual backend so the keyed /// index, the `key_index_backfill_watermark` table, and the materialization run as they -/// do in production: the 413 reproduces, the backfill watermarks despite duplicate -/// roots, a NEW workspace's root lookup is authoritative-absent (empty, NO 413 — the -/// soul write's exact path), and the existing absent-`ParentId` root resolves. +/// do in production: the backfill detects duplicate claims and withholds authority, +/// both a new workspace miss and an existing absent-`ParentId` hit remain scan-safe +/// until the duplicate conflict is repaired. #[test] fn directory_root_souls_scenario_on_postgres() { use temper_runtime::scheduler::sim_uuid as runtime_sim_uuid; @@ -369,22 +430,16 @@ fn directory_root_souls_scenario_on_postgres() { ("/", "/", "wsA", None), ) .await; - pg_seed_dir( - &state, - &tenant, - &agent_ctx, - "wsA-root-d1", - ("/", "/", "wsA", None), - ) - .await; - pg_seed_dir( - &state, - &tenant, - &agent_ctx, - "wsA-root-d2", - ("/", "/", "wsA", None), - ) - .await; + for duplicate in ["wsA-root-d1", "wsA-root-d2"] { + store + .append( + &format!("{tenant}:Directory:{duplicate}"), + 0, + &[legacy_directory_create("/", "/", "wsA", None)], + ) + .await + .expect("seed legacy duplicate root"); + } for i in 0..12 { pg_seed_dir( &state, @@ -413,13 +468,9 @@ fn directory_root_souls_scenario_on_postgres() { filter: Some(dir_root_filter("wsA")), ..QueryOptions::default() }; - // (1) Pre-watermark: a NEW workspace's root lookup misses. Historically this - // scanned 15 > budget → 413 (the original prod failure). The ARN-68 empty-list - // gap reconcile now bounds it: roots have NO ParentId field-index row, so they - // form the (small) coverage gap, get materialized, and `ParentId eq null` - // matches none for wsB → a bounded EMPTY answer instead of the 413. Were the - // gap larger than the budget (prod's 1688 duplicate roots), the 413 would - // remain — the keyed path below stays the authoritative fix for roots. + // (1) Pre-watermark: a NEW workspace's root lookup misses. A recognized + // exact-key query cannot use the asynchronous coverage-gap shortcut before + // v3 certification; the authoritative journal proof is 15 > budget → 413. match read_entity_set_page(QueryPlaneReadRequest { state: &state, tenant: &tenant, @@ -431,21 +482,19 @@ fn directory_root_souls_scenario_on_postgres() { }) .await { - Ok(result) => assert!( - result.entities.is_empty(), - "wsB has no root; the bounded gap reconcile must return empty" - ), - Err(_) => panic!("the gap reconcile must bound the pre-watermark root miss"), + Err(QueryPlaneReadError::QueryTooLarge { .. }) => {} + Ok(_) => panic!("pre-v3 root miss must not trust projection coverage"), + Err(_) => panic!("expected QueryTooLarge before the watermark"), } - // (2) Backfill on real Postgres → Directory watermarked (duplicate roots are - // key-conflict skips, not failures), and the absent-ParentId root is keyed. + // (2) Backfill on real Postgres → one absent-ParentId root is keyed, but the + // historical duplicate claims block the complete watermark. state.populate_key_index_from_snapshots(&tenant).await; assert!( - state - .key_index_backfill_complete(&tenant, "Directory", "name_parent") + !state + .key_index_backfill_complete(&tenant, "Directory", DIRECTORY_KEY_SET_SIGNATURE) .await, - "Directory must watermark on Postgres despite duplicate roots" + "Directory must remain non-authoritative until duplicates are repaired" ); assert!( store @@ -461,9 +510,10 @@ fn directory_root_souls_scenario_on_postgres() { "the wsA root (absent ParentId) is keyed on Postgres" ); - // (3a) The soul-write path: a NEW workspace (no root) → authoritative-absent → - // empty, NO 413 → ensure_dirs creates the root and the soul write proceeds. - let r = match read_entity_set_page(QueryPlaneReadRequest { + // (3a) A recognized exact-key miss bypasses the asynchronous projection. + // Since duplicate conflicts withheld v3 authority and the journal proof is + // larger than the budget, the safe answer is 413 rather than false absence. + match read_entity_set_page(QueryPlaneReadRequest { state: &state, tenant: &tenant, security_ctx: &security_ctx, @@ -474,20 +524,15 @@ fn directory_root_souls_scenario_on_postgres() { }) .await { - Ok(r) => r, - Err(_) => panic!("no 413 once watermarked — keyed miss is authoritative absence"), - }; - assert!( - r.entities.is_empty(), - "a workspace with no root resolves to absent" - ); - assert_eq!( - r.telemetry.fallback_reason, - QueryPlaneFallbackReason::KeyedAbsence - ); + Err(QueryPlaneReadError::QueryTooLarge { .. }) => {} + Ok(_) => panic!("conflicted backfill must not authorize key-index absence"), + Err(_) => panic!("expected scan-safe QueryTooLarge after conflict"), + } - // (3b) The existing absent-ParentId root resolves (no 413, exactly one). - let r = match read_entity_set_page(QueryPlaneReadRequest { + // (3b) The existing row is equally non-authoritative before v3. Trusting + // it would let a stale legacy claim bypass replay, so it must also honor + // the authoritative proof budget. + match read_entity_set_page(QueryPlaneReadRequest { state: &state, tenant: &tenant, security_ctx: &security_ctx, @@ -498,14 +543,10 @@ fn directory_root_souls_scenario_on_postgres() { }) .await { - Ok(r) => r, - Err(_) => panic!("existing root must resolve without 413"), - }; - assert_eq!( - r.entities.len(), - 1, - "wsA root resolves to the keyed root on Postgres" - ); + Err(QueryPlaneReadError::QueryTooLarge { .. }) => {} + Ok(_) => panic!("an incomplete keyed hit must not bypass the proof budget"), + Err(_) => panic!("expected scan-safe QueryTooLarge for incomplete hit"), + } }); } @@ -728,6 +769,37 @@ async fn key_index_backfill_keys_store_entities_absent_from_the_lazy_index() { .await .expect("seed snapshot"); } + // ADR-0077 migration shape: a snapshot can be the only durable source. It + // must participate in the same deterministic replay and coverage proof even + // though no journal key exists for enumeration. + let snapshot_only = serde_json::json!({ + "entity_type": "Order", + "entity_id": "ord-snapshot-only", + "status": "Draft", + "item_count": 0, + "fields": { + "Id": "ord-snapshot-only", + "Status": "Draft", + "WorkspaceId": "ws1", + "Path": "/snapshot-only" + }, + }); + store + .save_snapshot( + &format!("{tenant}:Order:ord-snapshot-only"), + 5, + &serde_json::to_vec(&snapshot_only).unwrap(), + ) + .await + .expect("seed snapshot-only migrated entity"); + assert!( + store + .read_events(&format!("{tenant}:Order:ord-snapshot-only"), 0) + .await + .expect("snapshot-only journal read") + .is_empty(), + "precondition: migrated snapshot owner has no journal" + ); // Nothing is keyed yet: the entities were seeded via `save_snapshot` with no // key-bearing Create event, so the sim store's live co-commit saw no key fields. @@ -757,11 +829,11 @@ async fn key_index_backfill_keys_store_entities_absent_from_the_lazy_index() { // Enumerated from the store and keyed both entities, and watermarked the type. assert!( state - .key_index_backfill_complete(&tenant, "Order", "ws_path") + .key_index_backfill_complete(&tenant, "Order", ORDER_KEY_SET_SIGNATURE) .await, "Order must be watermarked after a clean backfill" ); - for (ws, path) in [("ws1", "/a"), ("ws1", "/b")] { + for (ws, path) in [("ws1", "/a"), ("ws1", "/b"), ("ws1", "/snapshot-only")] { assert!( store .lookup_by_key(tenant.as_str(), "Order", "ws_path", &ws_path_hash(ws, path)) @@ -773,12 +845,11 @@ async fn key_index_backfill_keys_store_entities_absent_from_the_lazy_index() { } } -/// Robustness (ADR-0153): the backfill is RESUMABLE — already-keyed entities are -/// skipped (not re-loaded), so a re-run after a partial pass only processes the -/// remainder instead of re-loading all N. Pre-key one entity directly, then run the -/// backfill, and confirm it completes + watermarks with both entities keyed. +/// Robustness (ADR-0153/0171): an incomplete index cannot trust an existing row as +/// proof that an entity is current. Pre-key one entity directly, then run the exact +/// full pass and confirm it completes + watermarks with both entities keyed. #[tokio::test] -async fn key_index_backfill_skips_already_keyed_entities_and_still_watermarks() { +async fn key_index_backfill_reconciles_existing_rows_and_still_watermarks() { let (state, store) = build_order_state_with_sim("key-backfill-resume"); let tenant = TenantId::default(); let agent_ctx = AgentContext::for_service("resume-test"); @@ -808,11 +879,32 @@ async fn key_index_backfill_skips_already_keyed_entities_and_still_watermarks() .expect("snap"); } // Pre-key ord-a directly (a prior partial pass / co-commit already keyed it). + let ord_a_sequence = current_sequence(&store, &tenant, "Order", "ord-a").await; + let (ord_a_snapshot_sequence, ord_a_snapshot) = store + .load_snapshot(&format!("{tenant}:Order:ord-a")) + .await + .expect("load ord-a snapshot") + .expect("ord-a snapshot exists"); + let repair_revision = store + .begin_key_index_backfill(tenant.as_str(), "Order", ORDER_KEY_SET_SIGNATURE) + .await + .expect("begin pre-key repair"); store .backfill_entity_keys( tenant.as_str(), "Order", "ord-a", + ord_a_sequence, + KeyIndexBackfillFence { + key_set_signature: ORDER_KEY_SET_SIGNATURE, + contract_revision: repair_revision, + expected_journal_sequence: 1, + expected_entity_live: true, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: ord_a_snapshot_sequence, + state: &ord_a_snapshot, + }), + }, &[temper_runtime::persistence::EntityKeyRow { key_name: "ws_path".to_string(), key_hash: ws_path_hash("ws1", "/a"), @@ -823,10 +915,10 @@ async fn key_index_backfill_skips_already_keyed_entities_and_still_watermarks() state.populate_key_index_from_snapshots(&tenant).await; - // ord-a was skipped via the already-keyed set; ord-b keyed fresh; type watermarked. + // Both entities were reconciled from replayed state, then the type was watermarked. assert!( state - .key_index_backfill_complete(&tenant, "Order", "ws_path") + .key_index_backfill_complete(&tenant, "Order", ORDER_KEY_SET_SIGNATURE) .await ); for (ws, path) in [("ws1", "/a"), ("ws1", "/b")] { @@ -840,19 +932,17 @@ async fn key_index_backfill_skips_already_keyed_entities_and_still_watermarks() } } -/// Soundness (ADR-0153): a DELETED entity is correctly skipped (not keyed) and does -/// NOT block the watermark — only entities that exist-but-cannot-load do. A deleted -/// entity alongside a live one: the type still watermarks, the live one is keyed, the -/// deleted one is not. +/// Repair + soundness (ADR-0153/0171): even with NO prior watermark, a full pass +/// purges stale rows from both a deleted entity and a live entity whose current key +/// is all-null. An interrupted/pre-v3 index may already contain rows, but row presence +/// is not trusted as coverage. Neither entity blocks the watermark; the live keyed +/// entity remains addressable. #[tokio::test] async fn key_index_backfill_skips_deleted_entities_without_blocking_watermark() { let (state, store) = build_order_state_with_sim("key-backfill-deleted"); let tenant = TenantId::default(); let agent_ctx = AgentContext::for_service("deleted-test"); - for (eid, status, path) in [ - ("ord-live", "Draft", "/live"), - ("ord-del", "Deleted", "/del"), - ] { + for (eid, path) in [("ord-live", "/live"), ("ord-del", "/del")] { state .dispatch_tenant_action( &tenant, @@ -865,7 +955,7 @@ async fn key_index_backfill_skips_deleted_entities_without_blocking_watermark() .await .expect("create"); let snap = serde_json::json!({ - "entity_type": "Order", "entity_id": eid, "status": status, "item_count": 0, + "entity_type": "Order", "entity_id": eid, "status": "Draft", "item_count": 0, "fields": { "Id": eid, "WorkspaceId": "ws1", "Path": path }, }); store @@ -877,12 +967,129 @@ async fn key_index_backfill_skips_deleted_entities_without_blocking_watermark() .await .expect("snap"); } + state + .dispatch_tenant_action( + &tenant, + "Order", + "ord-null", + "Create", + serde_json::json!({}), + &agent_ctx, + ) + .await + .expect("create all-null entity"); + let null_snap = serde_json::json!({ + "entity_type": "Order", "entity_id": "ord-null", "status": "Draft", "item_count": 0, + "fields": { "Id": "ord-null", "WorkspaceId": null, "Path": null }, + }); + store + .save_snapshot( + &format!("{tenant}:Order:ord-null"), + 1, + &serde_json::to_vec(&null_snap).unwrap(), + ) + .await + .expect("all-null snap"); + + let deleted_stale = ws_path_hash("ws1", "/del"); + let all_null_stale = ws_path_hash("ws-old", "/old"); + let stale_signature = "pre-v3:ws_path"; + let stale_revision = store + .begin_key_index_backfill(tenant.as_str(), "Order", stale_signature) + .await + .expect("begin stale-row fixture contract"); + for (entity_id, key_hash) in [ + ("ord-del", deleted_stale.clone()), + ("ord-null", all_null_stale.clone()), + ] { + let sequence_nr = current_sequence(&store, &tenant, "Order", entity_id).await; + let (snapshot_sequence, snapshot_state) = store + .load_snapshot(&format!("{tenant}:Order:{entity_id}")) + .await + .expect("load fixture snapshot") + .expect("fixture snapshot exists"); + store + .backfill_entity_keys( + tenant.as_str(), + "Order", + entity_id, + sequence_nr, + KeyIndexBackfillFence { + key_set_signature: stale_signature, + contract_revision: stale_revision, + expected_journal_sequence: 1, + expected_entity_live: true, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: snapshot_sequence, + state: &snapshot_state, + }), + }, + &[temper_runtime::persistence::EntityKeyRow { + key_name: "ws_path".to_string(), + key_hash, + }], + ) + .await + .expect("seed stale pre-v3 row"); + } + // This is the pre-v3 crash shape: the stale ownership row exists before a + // tombstone append that does not participate in exact key reconciliation. + let deleted_at = temper_runtime::scheduler::sim_now(); + store + .append( + &format!("{tenant}:Order:ord-del"), + 1, + &[PersistenceEnvelope { + sequence_nr: 0, + event_type: "Deleted".to_string(), + payload: serde_json::json!({ + "action": "Deleted", + "from_status": "Draft", + "to_status": "Deleted", + "timestamp": deleted_at, + "params": {}, + }), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: deleted_at, + actor_id: "default:Order:ord-del".to_string(), + }, + }], + ) + .await + .expect("append terminal tombstone"); + let stale_live_snapshot = serde_json::json!({ + "entity_type": "Order", + "entity_id": "ord-del", + "status": "Draft", + "item_count": 0, + "total_event_count": 3, + "fields": { "Id": "ord-del", "WorkspaceId": "ws1", "Path": "/del" }, + }); + store + .save_snapshot( + &format!("{tenant}:Order:ord-del"), + 3, + &serde_json::to_vec(&stale_live_snapshot).unwrap(), + ) + .await + .expect("seed newer stale live snapshot after tombstone"); + assert!( + store + .key_index_backfilled_types(tenant.as_str()) + .await + .unwrap() + .is_empty(), + "precondition: stale rows exist without a completed watermark" + ); state.populate_key_index_from_snapshots(&tenant).await; assert!( state - .key_index_backfill_complete(&tenant, "Order", "ws_path") + .key_index_backfill_complete(&tenant, "Order", ORDER_KEY_SET_SIGNATURE) .await, "a deleted entity must not block the watermark" ); @@ -901,17 +1108,20 @@ async fn key_index_backfill_skips_deleted_entities_without_blocking_watermark() ); assert!( store - .lookup_by_key( - tenant.as_str(), - "Order", - "ws_path", - &ws_path_hash("ws1", "/del") - ) + .lookup_by_key(tenant.as_str(), "Order", "ws_path", &deleted_stale) .await .unwrap() .is_none(), "deleted entity is not keyed" ); + assert_eq!( + store + .lookup_by_key(tenant.as_str(), "Order", "ws_path", &all_null_stale) + .await + .unwrap(), + None, + "all-null current state must purge its stale prior key" + ); } /// Soundness gate (ADR-0153): an entity that EXISTS but whose journal cannot be read @@ -922,7 +1132,12 @@ async fn key_index_backfill_skips_deleted_entities_without_blocking_watermark() /// wrong-absent. #[tokio::test] async fn key_index_backfill_loadfailed_entity_blocks_watermark_then_resumes() { - let (state, store) = build_order_state_with_sim("key-backfill-loadfail"); + let store = SimEventStore::new(0, SimFaultConfig::none()); + let query_plane = std::sync::Arc::new(SimQueryPlane::default()); + let mut state = build_order_state("key-backfill-loadfail"); + let mut storage = StorageStack::from_sim(store.clone(), None); + storage.query_plane = Some(query_plane.clone()); + state.set_storage_stack(storage); let tenant = TenantId::default(); let agent_ctx = AgentContext::for_service("loadfail-test"); let pid = format!("{tenant}:Order:ord-x"); @@ -946,6 +1161,26 @@ async fn key_index_backfill_loadfailed_entity_blocks_watermark_then_resumes() { .save_snapshot(&pid, 1, &serde_json::to_vec(&snap).unwrap()) .await .expect("snap"); + let stale_catalog_fields = + serde_json::json!({ "Id": "ord-x", "WorkspaceId": "ws1", "Path": "/stale" }); + query_plane + .upsert_projection( + tenant.as_str(), + "Order", + "ord-x", + "Draft", + &stale_catalog_fields, + &serde_json::json!({ + "entity_type": "Order", + "entity_id": "ord-x", + "status": "Draft", + "fields": stale_catalog_fields, + "sequence_nr": 1, + }), + 1, + ) + .await + .expect("seed stale catalog fallback"); // Run 1: the entity's journal read fails → LoadFailed → type NOT watermarked, // entity NOT keyed. @@ -953,7 +1188,7 @@ async fn key_index_backfill_loadfailed_entity_blocks_watermark_then_resumes() { state.populate_key_index_from_snapshots(&tenant).await; assert!( !state - .key_index_backfill_complete(&tenant, "Order", "ws_path") + .key_index_backfill_complete(&tenant, "Order", ORDER_KEY_SET_SIGNATURE) .await, "an unloadable entity must block the watermark" ); @@ -970,12 +1205,25 @@ async fn key_index_backfill_loadfailed_entity_blocks_watermark_then_resumes() { .is_none(), "the unloadable entity must not be keyed" ); + assert!( + store + .lookup_by_key( + tenant.as_str(), + "Order", + "ws_path", + &ws_path_hash("ws1", "/stale") + ) + .await + .unwrap() + .is_none(), + "strict recovery failure must not be masked by stale catalog ownership" + ); // Run 2 (resume): the read now succeeds → entity keyed → type watermarked. state.populate_key_index_from_snapshots(&tenant).await; assert!( state - .key_index_backfill_complete(&tenant, "Order", "ws_path") + .key_index_backfill_complete(&tenant, "Order", ORDER_KEY_SET_SIGNATURE) .await, "backfill must resume and watermark once the read succeeds" ); @@ -994,6 +1242,87 @@ async fn key_index_backfill_loadfailed_entity_blocks_watermark_then_resumes() { ); } +/// Catalog compatibility is only valid when the journal is absent. A durable but +/// schema-incompatible event can advance the journal generation without producing +/// replayable fields; a same-generation catalog row must not become authoritative. +#[tokio::test] +async fn key_index_backfill_never_uses_catalog_for_journal_present_missing_state() { + let store = SimEventStore::new(0, SimFaultConfig::none()); + let query_plane = std::sync::Arc::new(SimQueryPlane::default()); + let mut state = build_order_state("key-backfill-journal-present-missing"); + let mut storage = StorageStack::from_sim(store.clone(), None); + storage.query_plane = Some(query_plane.clone()); + state.set_storage_stack(storage); + let tenant = TenantId::default(); + let persistence_id = format!("{tenant}:Order:ord-legacy"); + let timestamp = temper_runtime::scheduler::sim_now(); + + store + .append( + &persistence_id, + 0, + &[PersistenceEnvelope { + sequence_nr: 0, + event_type: "LegacyCreate".to_string(), + payload: serde_json::json!({ "legacy_schema": true }), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: persistence_id.clone(), + }, + }], + ) + .await + .expect("seed incompatible journal event"); + let stale_catalog_fields = serde_json::json!({ + "Id": "ord-legacy", + "WorkspaceId": "ws1", + "Path": "/stale-catalog" + }); + query_plane + .upsert_projection( + tenant.as_str(), + "Order", + "ord-legacy", + "Draft", + &stale_catalog_fields, + &serde_json::json!({ + "entity_type": "Order", + "entity_id": "ord-legacy", + "status": "Draft", + "fields": stale_catalog_fields, + "sequence_nr": 1, + }), + 1, + ) + .await + .expect("seed same-generation stale catalog row"); + + state.populate_key_index_from_snapshots(&tenant).await; + + assert!( + !state + .key_index_backfill_complete(&tenant, "Order", ORDER_KEY_SET_SIGNATURE) + .await, + "journal-present missing state must block authoritative coverage" + ); + assert_eq!( + store + .lookup_by_key( + tenant.as_str(), + "Order", + "ws_path", + &ws_path_hash("ws1", "/stale-catalog"), + ) + .await + .expect("lookup stale catalog key"), + None, + "journal-present missing state must not install stale catalog ownership" + ); +} + /// C (ADR-0153): once the backfill watermark is set, a keyed read MISS is /// authoritative absence — the read returns empty WITHOUT the full-type scan that /// otherwise 413s at scale (ARN-68). Before the watermark, the same miss falls back @@ -1036,7 +1365,7 @@ async fn keyed_miss_returns_empty_without_scan_413_once_watermarked() { // Watermark Order → a keyed miss is now authoritative absence. state - .mark_key_index_backfilled(&tenant, "Order", "ws_path") + .mark_key_index_backfilled(&tenant, "Order", ORDER_KEY_SET_SIGNATURE) .await; let result = match read_entity_set_page(QueryPlaneReadRequest { @@ -1077,9 +1406,9 @@ async fn key_index_backfill_rekeys_existing_entities_when_a_key_is_added() { // Two orders that ALREADY have a key row under an OLDER key name, and whose // ws_path-valued fields live in the snapshot — the exact prod shape: entities keyed // under an earlier declaration (here `old_key`), the new key (`ws_path`) not yet - // assigned. The old-key rows put them in `keyed_entity_ids_for_type`, so the - // per-entity resumability skip WOULD skip them — this is what `force_full_rekey` - // must bypass. Without the bypass this test fails (ws_path never gets assigned). + // assigned. The old-key rows put them in `keyed_entity_ids_for_type`, proving why + // row presence cannot be treated as complete coverage. Without a full exact pass, + // ws_path would never be assigned. for (eid, ws, path) in [("ord-rk-0", "ws1", "/a"), ("ord-rk-1", "ws1", "/b")] { state .dispatch_tenant_action( @@ -1104,12 +1433,30 @@ async fn key_index_backfill_rekeys_existing_entities_when_a_key_is_added() { ) .await .expect("seed snapshot"); - // Key it under the OLD key only (so it appears already-keyed for resumability). + let snapshot_bytes = serde_json::to_vec(&snapshot).unwrap(); + // Key it under the OLD key only (so row presence cannot be mistaken for + // complete coverage of the new declaration). + let old_signature = "old_key"; + let old_revision = store + .begin_key_index_backfill(tenant.as_str(), "Order", old_signature) + .await + .expect("begin old-key fixture contract"); store .backfill_entity_keys( tenant.as_str(), "Order", eid, + current_sequence(&store, &tenant, "Order", eid).await, + KeyIndexBackfillFence { + key_set_signature: old_signature, + contract_revision: old_revision, + expected_journal_sequence: 1, + expected_entity_live: true, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: 1, + state: &snapshot_bytes, + }), + }, &[temper_runtime::persistence::EntityKeyRow { key_name: "old_key".to_string(), key_hash: format!("old-hash-{eid}"), @@ -1130,19 +1477,19 @@ async fn key_index_backfill_rekeys_existing_entities_when_a_key_is_added() { // INCOMPLETE for ws_path — a ws_path miss falls back to the scan, never a wrong absent. assert!( !state - .key_index_backfill_complete(&tenant, "Order", "ws_path") + .key_index_backfill_complete(&tenant, "Order", ORDER_KEY_SET_SIGNATURE) .await, "a stale watermark covering a different key-set must read as incomplete for the new key" ); - // The entities ARE already-keyed (under old_key) — so the resumability skip would - // exclude them; only force_full_rekey re-processes them. + // The entities ARE already-keyed (under old_key), but row presence cannot exclude + // them from an incomplete type's exact repair. assert!( !store .keyed_entity_ids_for_type(tenant.as_str(), "Order") .await .unwrap() .is_empty(), - "precondition: entities appear already-keyed (old_key), so the resume-skip would skip them" + "precondition: entities already have rows under old_key" ); assert!( store @@ -1158,13 +1505,13 @@ async fn key_index_backfill_rekeys_existing_entities_when_a_key_is_added() { "precondition: not yet keyed for the newly-added ws_path key" ); - // Re-run the backfill: covered != current declared → force-full re-key of every + // Re-run the backfill: covered != current declared → exact full repair of every // existing entity, then re-watermark with the current key-set. state.populate_key_index_from_snapshots(&tenant).await; assert!( state - .key_index_backfill_complete(&tenant, "Order", "ws_path") + .key_index_backfill_complete(&tenant, "Order", ORDER_KEY_SET_SIGNATURE) .await, "after the re-key the type is complete for the current declared key-set" ); @@ -1182,20 +1529,21 @@ async fn key_index_backfill_rekeys_existing_entities_when_a_key_is_added() { } #[test] -fn declared_key_set_signature_is_sorted_and_joined() { +fn declared_key_set_signature_is_versioned_sorted_and_complete() { use temper_jit::table::types::DeclaredKey; let key = |name: &str| DeclaredKey { name: name.to_string(), properties: vec!["WorkspaceId".to_string(), "Path".to_string()], }; - // Order-independent, comma-joined, sorted by name. + // Versioned, declaration-order independent, and complete through each ordered + // property list. assert_eq!( crate::key_index::declared_key_set_signature(&[key("ws_path"), key("name_parent")]), - "name_parent,ws_path" + "v3|11:name_parent[11:WorkspaceId4:Path]|7:ws_path[11:WorkspaceId4:Path]" ); - assert_eq!(crate::key_index::declared_key_set_signature(&[]), ""); + assert_eq!(crate::key_index::declared_key_set_signature(&[]), "v3"); assert_eq!( crate::key_index::declared_key_set_signature(&[key("only")]), - "only" + "v3|4:only[11:WorkspaceId4:Path]" ); } diff --git a/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/backend_authority.rs b/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/backend_authority.rs new file mode 100644 index 000000000..214a5d4c1 --- /dev/null +++ b/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/backend_authority.rs @@ -0,0 +1,183 @@ +//! Backend-capability regressions for the declared-key absence oracle. + +use super::*; +use temper_runtime::persistence::{EventMetadata, EventStore, PersistenceEnvelope}; +use temper_runtime::scheduler::{sim_now, sim_uuid}; + +/// Turso can probe legacy key rows but does not co-commit or repair them. A no-op +/// backfill must therefore never make a keyed miss authoritative in the server's +/// in-memory watermark cache, even when the type currently has no entities. +#[tokio::test] +async fn non_authoritative_turso_backfill_never_marks_key_index_complete() { + let store = TursoEventStore::new("file::memory:", None) + .await + .expect("create in-memory Turso store"); + let mut state = build_order_state("turso-key-authority"); + state.set_storage_stack(StorageStack::from_turso(store)); + let tenant = TenantId::default(); + + state.populate_key_index_from_snapshots(&tenant).await; + + assert!( + !state + .key_index_backfill_complete(&tenant, "Order", super::super::ORDER_KEY_SET_SIGNATURE) + .await, + "a backend without co-committed exact key reconciliation must remain scan-safe" + ); +} + +fn exact_key_filter(workspace: &str, path: &str) -> FilterExpr { + FilterExpr::BinaryOp { + left: Box::new(FilterExpr::BinaryOp { + left: Box::new(FilterExpr::Property("WorkspaceId".to_string())), + op: BinaryOperator::Eq, + right: Box::new(FilterExpr::Literal(ODataValue::String( + workspace.to_string(), + ))), + }), + op: BinaryOperator::And, + right: Box::new(FilterExpr::BinaryOp { + left: Box::new(FilterExpr::Property("Path".to_string())), + op: BinaryOperator::Eq, + right: Box::new(FilterExpr::Literal(ODataValue::String(path.to_string()))), + }), + } +} + +/// Recognizing an exact declared key is itself an authority boundary. A backend +/// without co-committed key ownership must scan the journal rather than falling +/// back to a stale native/catalog projection. +#[tokio::test] +async fn non_authoritative_turso_exact_key_read_ignores_stale_live_projection() { + let db_url = format!("file:/tmp/temper-arn238-turso-authority-{}.db", sim_uuid()); + let store = TursoEventStore::new(&db_url, None) + .await + .expect("create in-memory Turso store"); + let mut state = build_order_state("turso-exact-key-authority"); + state.set_storage_stack(StorageStack::from_turso(store.clone())); + let tenant = TenantId::default(); + let entity_id = "turso-legacy-deleted"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let fields = serde_json::json!({ + "Id": entity_id, + "Status": "Draft", + "WorkspaceId": "ws-turso", + "Path": "/stale" + }); + let created_at = sim_now(); + let created = crate::entity_actor::EntityEvent { + action: "Created".to_string(), + from_status: String::new(), + to_status: "Draft".to_string(), + timestamp: created_at, + params: fields.clone(), + idempotency_key: None, + }; + store + .append( + &persistence_id, + 0, + &[PersistenceEnvelope { + sequence_nr: 1, + event_type: "Created".to_string(), + payload: serde_json::to_value(created).expect("serialize create"), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: created_at, + actor_id: persistence_id.clone(), + }, + }], + ) + .await + .expect("seed durable create"); + let stale_state = serde_json::json!({ + "entity_type": "Order", + "entity_id": entity_id, + "status": "Draft", + "fields": fields, + "sequence_nr": 1, + "events": [], + }); + store + .upsert_query_projection_with_state( + tenant.as_str(), + "Order", + entity_id, + "Draft", + &stale_state["fields"], + &stale_state, + 1, + ) + .await + .expect("seed stale live projection"); + let timestamp = sim_now(); + let deleted = crate::entity_actor::EntityEvent { + action: "Deleted".to_string(), + from_status: "Draft".to_string(), + to_status: "Deleted".to_string(), + timestamp, + params: serde_json::json!({}), + idempotency_key: None, + }; + store + .append( + &persistence_id, + 1, + &[PersistenceEnvelope { + sequence_nr: 2, + event_type: "Deleted".to_string(), + payload: serde_json::to_value(deleted).expect("serialize tombstone"), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: persistence_id.clone(), + }, + }], + ) + .await + .expect("append event-only tombstone"); + + let security_ctx = SecurityContext::system(); + let budget = QueryPlaneReadBudget { + default_page_size: 10, + max_entities: 10, + }; + for (count, ordered) in [(false, false), (true, false), (false, true)] { + let options = QueryOptions { + filter: Some(exact_key_filter("ws-turso", "/stale")), + count: count.then_some(true), + orderby: ordered.then(|| { + vec![OrderByClause { + property: "Path".to_string(), + direction: OrderDirection::Asc, + }] + }), + ..QueryOptions::default() + }; + let result = read_entity_set_page(QueryPlaneReadRequest { + state: &state, + tenant: &tenant, + security_ctx: &security_ctx, + entity_type: "Order", + entity_set_name: "Orders", + query_options: &options, + budget, + }) + .await; + let result = match result { + Ok(result) => result, + Err(_) => panic!("exact-key read remains bounded"), + }; + assert!( + result.entities.is_empty(), + "count={count}, ordered={ordered}: durable tombstone must beat stale Turso projection" + ); + if count { + assert_eq!(result.count, Some(0)); + } + } +} diff --git a/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/backfill_sources.rs b/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/backfill_sources.rs new file mode 100644 index 000000000..d864c4b25 --- /dev/null +++ b/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/backfill_sources.rs @@ -0,0 +1,279 @@ +use super::*; +use temper_runtime::persistence::{EntityKeyRow, EventMetadata, IndexReconciliation}; +use temper_runtime::scheduler::{install_deterministic_context, sim_now, sim_uuid}; + +mod snapshot_rewrite; + +fn legacy_snapshot(entity_id: &str, workspace: &str, path: &str) -> Vec { + serde_json::to_vec(&serde_json::json!({ + "entity_type": "Order", + "entity_id": entity_id, + "status": "Draft", + "item_count": 0, + "fields": { + "Id": entity_id, + "Status": "Draft", + "WorkspaceId": workspace, + "Path": path, + }, + })) + .expect("serialize legacy snapshot") +} + +fn journal_path_delta(persistence_id: &str, path: &str, token: &str) -> PersistenceEnvelope { + let timestamp = sim_now(); + PersistenceEnvelope { + sequence_nr: 0, + event_type: "Temper.Internal.FieldUpdate.v1".to_string(), + payload: serde_json::json!({ + "schema": "temper.field-update.v1", + "fields": {"Path": path}, + "replace": false, + "idempotency_key": token, + }), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: persistence_id.to_string(), + }, + } +} + +async fn read_exact_path( + state: &ServerState, + tenant: &TenantId, + workspace: &str, + path: &str, +) -> Result { + let options = QueryOptions { + filter: Some(ws_path_filter(workspace, path)), + ..QueryOptions::default() + }; + let security_ctx = SecurityContext::system(); + read_entity_set_page(QueryPlaneReadRequest { + state, + tenant, + security_ctx: &security_ctx, + entity_type: "Order", + entity_set_name: "Orders", + query_options: &options, + budget: QueryPlaneReadBudget { + default_page_size: 10, + max_entities: 10, + }, + }) + .await +} + +fn require_read( + result: Result, + message: &str, +) -> QueryPlaneReadResult { + match result { + Ok(result) => result, + Err(_) => panic!("{message}"), + } +} + +#[tokio::test] +async fn key_backfill_prefers_journal_over_equal_sequence_snapshot_source() { + let (_guard, _clock, _ids) = install_deterministic_context(265); + let tenant = TenantId::default(); + let workspace = "ws-backfill-source"; + let snapshot_path = "/snapshot-generation"; + let journal_path = "/journal-generation"; + let entity_id = "ord-backfill-equal-sequence-source"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let (state, store) = build_order_state_with_sim("backfill-source-replacement"); + + let snapshot = serde_json::json!({ + "entity_type": "Order", + "entity_id": entity_id, + "status": "Draft", + "item_count": 0, + "fields": { + "Id": entity_id, + "Status": "Draft", + "WorkspaceId": workspace, + "Path": snapshot_path, + }, + }); + EventStore::save_snapshot( + &store, + &persistence_id, + 1, + &serde_json::to_vec(&snapshot).expect("serialize stale snapshot generation"), + ) + .await + .expect("seed snapshot-only generation"); + + let timestamp = sim_now(); + EventStore::append_with_index_rows( + &store, + &persistence_id, + 0, + &[PersistenceEnvelope { + sequence_nr: 0, + event_type: "Temper.Internal.FieldUpdate.v1".to_string(), + payload: serde_json::json!({ + "schema": "temper.field-update.v1", + "fields": { + "Id": entity_id, + "WorkspaceId": workspace, + "Path": journal_path, + }, + "replace": false, + "idempotency_key": "backfill-source-replacement", + }), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: persistence_id.clone(), + }, + }], + &[EntityKeyRow { + key_name: "ws_path".to_string(), + key_hash: ws_path_hash(workspace, journal_path), + }], + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(ORDER_KEY_SET_SIGNATURE.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .expect("replace snapshot-only source with an equal-sequence journal generation"); + + state.populate_key_index_from_snapshots(&tenant).await; + + assert!( + state + .key_index_backfill_complete(&tenant, "Order", ORDER_KEY_SET_SIGNATURE) + .await, + "the stable journal generation must permit coverage publication" + ); + assert_eq!( + EventStore::lookup_by_key( + &store, + tenant.as_str(), + "Order", + "ws_path", + &ws_path_hash(workspace, journal_path), + ) + .await + .expect("lookup journal-derived ownership") + .as_deref(), + Some(entity_id), + "full backfill must preserve the current journal-derived key" + ); + assert_eq!( + EventStore::lookup_by_key( + &store, + tenant.as_str(), + "Order", + "ws_path", + &ws_path_hash(workspace, snapshot_path), + ) + .await + .expect("lookup stale snapshot ownership"), + None, + "an equal-sequence stale snapshot must never replace journal ownership" + ); +} + +#[tokio::test] +async fn complete_key_read_uses_the_same_legacy_snapshot_baseline_as_backfill() { + let (_guard, _clock, _ids) = install_deterministic_context(267); + let tenant = TenantId::default(); + let workspace = "ws-hybrid-source"; + let journal_path = "/journal-delta"; + let entity_id = "ord-hybrid-source"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let (state, store) = build_order_state_with_sim("backfill-read-source-parity"); + + EventStore::save_snapshot( + &store, + &persistence_id, + 1, + &legacy_snapshot(entity_id, workspace, "/snapshot-baseline"), + ) + .await + .expect("seed legacy fields that are absent from the journal delta"); + EventStore::append( + &store, + &persistence_id, + 0, + &[journal_path_delta( + &persistence_id, + journal_path, + "hybrid-source-delta", + )], + ) + .await + .expect("seed journal generation with only the changed key component"); + + state.populate_key_index_from_snapshots(&tenant).await; + assert!( + state + .key_index_backfill_complete(&tenant, "Order", ORDER_KEY_SET_SIGNATURE) + .await, + "stable hybrid source must publish coverage" + ); + + let result = require_read( + read_exact_path(&state, &tenant, workspace, journal_path).await, + "a covered owner must remain materializable from the source used by backfill", + ); + assert_eq!(result.entities.len(), 1); + assert_eq!(result.entities[0]["entity_id"], entity_id); + assert_eq!(result.entities[0]["fields"]["WorkspaceId"], workspace); + assert_eq!(result.entities[0]["fields"]["Path"], journal_path); +} + +#[tokio::test] +async fn complete_key_read_materializes_snapshot_only_owner_without_bootstrapping_journal() { + let (_guard, _clock, _ids) = install_deterministic_context(268); + let tenant = TenantId::default(); + let workspace = "ws-snapshot-owner"; + let path = "/snapshot-only-owner"; + let entity_id = "ord-snapshot-only-owner"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let (state, store) = build_order_state_with_sim("complete-snapshot-only-owner"); + + EventStore::save_snapshot( + &store, + &persistence_id, + 5, + &legacy_snapshot(entity_id, workspace, path), + ) + .await + .expect("seed ADR-0077 snapshot-only owner"); + state.populate_key_index_from_snapshots(&tenant).await; + assert!( + state + .key_index_backfill_complete(&tenant, "Order", ORDER_KEY_SET_SIGNATURE) + .await, + "snapshot-only owner must be included in complete coverage" + ); + + let result = require_read( + read_exact_path(&state, &tenant, workspace, path).await, + "snapshot-only indexed owner must remain readable", + ); + assert_eq!(result.entities.len(), 1); + assert_eq!(result.entities[0]["entity_id"], entity_id); + assert_eq!( + EventStore::journal_boundary(&store, &persistence_id) + .await + .expect("journal remains readable") + .latest_sequence, + 0, + "materializing a snapshot-only owner must not bootstrap a journal" + ); +} diff --git a/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/backfill_sources/snapshot_rewrite.rs b/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/backfill_sources/snapshot_rewrite.rs new file mode 100644 index 000000000..381ea5542 --- /dev/null +++ b/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/backfill_sources/snapshot_rewrite.rs @@ -0,0 +1,596 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + +use super::*; +use crate::storage::{BackendLabel, BoxedEventStore, StorageStack}; +use temper_runtime::persistence::{ + KeyIndexBackfillFence, PersistenceAppend, PersistenceAppendResult, PersistenceError, +}; + +#[derive(Clone)] +struct SnapshotRewriteDuringBackfillStore { + inner: SimEventStore, + persistence_id: String, + replacement: Vec, + rewritten: Arc, + rewrite_on_snapshot_load: bool, + rewrite_on_append: bool, + append_attempts: Arc, +} + +impl EventStore for SnapshotRewriteDuringBackfillStore { + fn supports_authoritative_key_index(&self) -> bool { + true + } + + async fn append( + &self, + persistence_id: &str, + expected_sequence: u64, + events: &[PersistenceEnvelope], + ) -> Result { + EventStore::append(&self.inner, persistence_id, expected_sequence, events).await + } + + async fn append_with_index_rows( + &self, + persistence_id: &str, + expected_sequence: u64, + events: &[PersistenceEnvelope], + key_rows: &[EntityKeyRow], + vector_rows: &[temper_runtime::persistence::EntityVectorRow], + reconciliation: IndexReconciliation, + ) -> Result { + if persistence_id == self.persistence_id { + self.append_attempts.fetch_add(1, Ordering::SeqCst); + if self.rewrite_on_append && !self.rewritten.swap(true, Ordering::SeqCst) { + let sequence_nr = EventStore::load_snapshot(&self.inner, persistence_id) + .await? + .map(|(sequence_nr, _)| sequence_nr) + .expect("append rewrite fixture requires a captured snapshot"); + EventStore::save_snapshot( + &self.inner, + persistence_id, + sequence_nr, + &self.replacement, + ) + .await?; + } + } + EventStore::append_with_index_rows( + &self.inner, + persistence_id, + expected_sequence, + events, + key_rows, + vector_rows, + reconciliation, + ) + .await + } + + async fn append_batch( + &self, + appends: &[PersistenceAppend], + ) -> Result, PersistenceError> { + EventStore::append_batch(&self.inner, appends).await + } + + async fn read_events( + &self, + persistence_id: &str, + from_sequence: u64, + ) -> Result, PersistenceError> { + EventStore::read_events(&self.inner, persistence_id, from_sequence).await + } + + async fn read_events_page( + &self, + persistence_id: &str, + from_sequence: u64, + through_sequence: u64, + limit: usize, + ) -> Result, PersistenceError> { + EventStore::read_events_page( + &self.inner, + persistence_id, + from_sequence, + through_sequence, + limit, + ) + .await + } + + async fn save_snapshot( + &self, + persistence_id: &str, + sequence_nr: u64, + snapshot: &[u8], + ) -> Result<(), PersistenceError> { + EventStore::save_snapshot(&self.inner, persistence_id, sequence_nr, snapshot).await + } + + async fn load_snapshot( + &self, + persistence_id: &str, + ) -> Result)>, PersistenceError> { + let captured = EventStore::load_snapshot(&self.inner, persistence_id).await?; + if self.rewrite_on_snapshot_load + && persistence_id == self.persistence_id + && !self.rewritten.swap(true, Ordering::SeqCst) + { + let sequence_nr = captured + .as_ref() + .map(|(sequence_nr, _)| *sequence_nr) + .expect("rewrite fixture requires a captured snapshot"); + EventStore::save_snapshot(&self.inner, persistence_id, sequence_nr, &self.replacement) + .await?; + } + Ok(captured) + } + + async fn list_entity_ids( + &self, + tenant: &str, + ) -> Result, PersistenceError> { + EventStore::list_entity_ids(&self.inner, tenant).await + } + + async fn list_entity_ids_by_type( + &self, + tenant: &str, + entity_type: &str, + ) -> Result, PersistenceError> { + EventStore::list_entity_ids_by_type(&self.inner, tenant, entity_type).await + } + + async fn list_entity_ids_for_key_reconciliation( + &self, + tenant: &str, + entity_type: &str, + ) -> Result, PersistenceError> { + EventStore::list_entity_ids_for_key_reconciliation(&self.inner, tenant, entity_type).await + } + + async fn list_key_reconciliation_page( + &self, + tenant: &str, + entity_type: &str, + after_entity_id: Option<&str>, + through_entity_id: &str, + limit: usize, + ) -> Result, PersistenceError> { + EventStore::list_key_reconciliation_page( + &self.inner, + tenant, + entity_type, + after_entity_id, + through_entity_id, + limit, + ) + .await + } + + async fn backfill_entity_keys( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + expected_sequence: u64, + contract_fence: KeyIndexBackfillFence<'_>, + key_rows: &[EntityKeyRow], + ) -> Result<(), PersistenceError> { + if !self.rewritten.swap(true, Ordering::SeqCst) { + let sequence_nr = EventStore::load_snapshot(&self.inner, &self.persistence_id) + .await? + .map(|(sequence_nr, _)| sequence_nr) + .expect("rewrite fixture requires a captured snapshot"); + EventStore::save_snapshot( + &self.inner, + &self.persistence_id, + sequence_nr, + &self.replacement, + ) + .await?; + } + EventStore::backfill_entity_keys( + &self.inner, + tenant, + entity_type, + entity_id, + expected_sequence, + contract_fence, + key_rows, + ) + .await + } + + async fn lookup_by_key( + &self, + tenant: &str, + entity_type: &str, + key_name: &str, + key_hash: &str, + ) -> Result, PersistenceError> { + EventStore::lookup_by_key(&self.inner, tenant, entity_type, key_name, key_hash).await + } + + async fn key_index_backfilled_types( + &self, + tenant: &str, + ) -> Result, PersistenceError> { + EventStore::key_index_backfilled_types(&self.inner, tenant).await + } + + async fn key_index_reconciliation_revision( + &self, + tenant: &str, + entity_type: &str, + ) -> Result { + EventStore::key_index_reconciliation_revision(&self.inner, tenant, entity_type).await + } + + async fn begin_key_index_backfill( + &self, + tenant: &str, + entity_type: &str, + key_set: &str, + ) -> Result { + EventStore::begin_key_index_backfill(&self.inner, tenant, entity_type, key_set).await + } + + async fn mark_key_index_backfilled_if_revision( + &self, + tenant: &str, + entity_type: &str, + key_set: &str, + expected_revision: u64, + ) -> Result { + EventStore::mark_key_index_backfilled_if_revision( + &self.inner, + tenant, + entity_type, + key_set, + expected_revision, + ) + .await + } +} + +#[tokio::test] +async fn same_sequence_snapshot_rewrite_invalidates_the_backfill_row() { + let (_guard, _clock, _ids) = install_deterministic_context(269); + let tenant = TenantId::default(); + let before_workspace = "ws-before-rewrite"; + let after_workspace = "ws-after-rewrite"; + let journal_path = "/journal-path"; + let entity_id = "ord-snapshot-rewrite"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let inner = SimEventStore::no_faults(269); + + EventStore::save_snapshot( + &inner, + &persistence_id, + 1, + &legacy_snapshot(entity_id, before_workspace, "/snapshot-path"), + ) + .await + .expect("seed snapshot baseline before repair"); + EventStore::append( + &inner, + &persistence_id, + 0, + &[journal_path_delta( + &persistence_id, + journal_path, + "snapshot-rewrite-delta", + )], + ) + .await + .expect("seed journal delta at the same numeric sequence"); + + let store = SnapshotRewriteDuringBackfillStore { + inner: inner.clone(), + persistence_id: persistence_id.clone(), + replacement: legacy_snapshot(entity_id, after_workspace, "/snapshot-path"), + rewritten: Arc::new(AtomicBool::new(false)), + rewrite_on_snapshot_load: false, + rewrite_on_append: false, + append_attempts: Arc::new(AtomicUsize::new(0)), + }; + let mut state = build_order_state("snapshot-rewrite-backfill"); + state.set_storage_stack(StorageStack::new( + BackendLabel::Sim, + BoxedEventStore::new(store), + None, + None, + None, + None, + None, + None, + None, + None, + )); + + state.populate_key_index_from_snapshots(&tenant).await; + assert!( + !state + .key_index_backfill_complete(&tenant, "Order", ORDER_KEY_SET_SIGNATURE) + .await, + "a same-sequence snapshot content change must reject the captured repair row" + ); + assert_eq!( + EventStore::lookup_by_key( + &inner, + tenant.as_str(), + "Order", + "ws_path", + &ws_path_hash(before_workspace, journal_path), + ) + .await + .expect("lookup stale mixed ownership"), + None, + "the pre-rewrite snapshot component must not be certified" + ); + + state.populate_key_index_from_snapshots(&tenant).await; + assert!( + state + .key_index_backfill_complete(&tenant, "Order", ORDER_KEY_SET_SIGNATURE) + .await, + "the next pass must converge on the stable replacement snapshot" + ); + assert_eq!( + EventStore::lookup_by_key( + &inner, + tenant.as_str(), + "Order", + "ws_path", + &ws_path_hash(after_workspace, journal_path), + ) + .await + .expect("lookup stable replacement ownership") + .as_deref(), + Some(entity_id) + ); +} + +#[tokio::test] +async fn actor_recovery_retries_a_same_sequence_snapshot_rewrite_before_upgrade() { + let (_guard, _clock, _ids) = install_deterministic_context(279); + let tenant = TenantId::default(); + let entity_id = "ord-actor-snapshot-rewrite"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let before_workspace = "ws-before-actor-rewrite"; + let after_workspace = "ws-after-actor-rewrite"; + let journal_path = "/journal-generation"; + let (state, inner) = build_order_state_with_sim("actor-snapshot-rewrite"); + EventStore::save_snapshot( + &inner, + &persistence_id, + 1, + &legacy_snapshot(entity_id, before_workspace, "/snapshot-generation"), + ) + .await + .expect("seed captured actor snapshot"); + EventStore::append( + &inner, + &persistence_id, + 0, + &[journal_path_delta( + &persistence_id, + journal_path, + "actor-snapshot-rewrite", + )], + ) + .await + .expect("seed equal-sequence journal generation"); + + let rewritten = Arc::new(AtomicBool::new(false)); + let store = BoxedEventStore::new(SnapshotRewriteDuringBackfillStore { + inner, + persistence_id, + replacement: legacy_snapshot(entity_id, after_workspace, "/snapshot-generation"), + rewritten: rewritten.clone(), + rewrite_on_snapshot_load: true, + rewrite_on_append: false, + append_attempts: Arc::new(AtomicUsize::new(0)), + }); + let table = state + .registry + .read() + .expect("registry lock") + .get_table_live(&tenant, "Order") + .expect("Order transition table") + .read() + .expect("table lock") + .clone(); + let recovered = crate::entity_actor::recover_entity_state_from_store( + crate::entity_actor::EntityRecoveryContext { + tenant: tenant.as_str(), + entity_type: "Order", + entity_id, + table: &table, + store: &store, + backend: BackendLabel::Sim, + initial_fields: &serde_json::json!({}), + blob_store: None, + }, + false, + ) + .await + .expect("recover one closed snapshot/journal generation"); + + assert!(rewritten.load(Ordering::SeqCst)); + assert_eq!( + recovered.fields["WorkspaceId"], after_workspace, + "recovery must retry the replacement snapshot instead of overwriting it with a stale provenance upgrade" + ); + assert_eq!(recovered.fields["Path"], journal_path); +} + +async fn actor_append_rewrite_fixture( + seed: u64, + entity_id: &str, + system_name: &str, +) -> ( + ServerState, + SimEventStore, + Arc, + Arc, +) { + let tenant = TenantId::default(); + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let inner = SimEventStore::no_faults(seed); + EventStore::save_snapshot( + &inner, + &persistence_id, + 1, + &legacy_snapshot(entity_id, "ws-before-append", "/journal-generation"), + ) + .await + .expect("seed actor snapshot before append race"); + EventStore::append( + &inner, + &persistence_id, + 0, + &[journal_path_delta( + &persistence_id, + "/journal-generation", + "append-race-journal-generation", + )], + ) + .await + .expect("seed equal-sequence journal generation"); + + let rewritten = Arc::new(AtomicBool::new(false)); + let append_attempts = Arc::new(AtomicUsize::new(0)); + let store = SnapshotRewriteDuringBackfillStore { + inner: inner.clone(), + persistence_id, + replacement: legacy_snapshot(entity_id, "ws-after-append", "/journal-generation"), + rewritten: rewritten.clone(), + rewrite_on_snapshot_load: false, + rewrite_on_append: true, + append_attempts: append_attempts.clone(), + }; + let mut state = build_order_state(system_name); + state.set_storage_stack(StorageStack::new( + BackendLabel::Sim, + BoxedEventStore::new(store), + None, + None, + None, + None, + None, + None, + None, + None, + )); + (state, inner, rewritten, append_attempts) +} + +#[tokio::test] +async fn field_update_retries_when_snapshot_rewrites_inside_the_append_boundary() { + let (_guard, _clock, _ids) = install_deterministic_context(280); + let tenant = TenantId::default(); + let entity_id = "ord-field-update-append-rewrite"; + let (state, inner, rewritten, append_attempts) = + actor_append_rewrite_fixture(280, entity_id, "field-update-append-rewrite").await; + + let updated = state + .update_tenant_entity_fields( + &tenant, + "Order", + entity_id, + serde_json::json!({"Path": "/after-field-update"}), + false, + Some("field-update-append-rewrite".to_string()), + ) + .await + .expect("field update must recover and retry the rewritten generation"); + + assert!(rewritten.load(Ordering::SeqCst)); + assert_eq!(append_attempts.load(Ordering::SeqCst), 2); + assert_eq!(updated.state.fields["WorkspaceId"], "ws-after-append"); + assert_eq!(updated.state.fields["Path"], "/after-field-update"); + assert_eq!( + EventStore::lookup_by_key( + &inner, + tenant.as_str(), + "Order", + "ws_path", + &ws_path_hash("ws-after-append", "/after-field-update"), + ) + .await + .expect("lookup recovered field-update ownership") + .as_deref(), + Some(entity_id) + ); + assert_eq!( + EventStore::lookup_by_key( + &inner, + tenant.as_str(), + "Order", + "ws_path", + &ws_path_hash("ws-before-append", "/after-field-update"), + ) + .await + .expect("lookup stale field-update ownership"), + None + ); +} + +#[tokio::test] +async fn domain_action_retries_when_snapshot_rewrites_inside_the_append_boundary() { + let (_guard, _clock, _ids) = install_deterministic_context(281); + let tenant = TenantId::default(); + let entity_id = "ord-domain-action-append-rewrite"; + let (state, inner, rewritten, append_attempts) = + actor_append_rewrite_fixture(281, entity_id, "domain-action-append-rewrite").await; + let agent = AgentContext::for_service("arn238-domain-action-append-rewrite"); + + let updated = state + .dispatch_tenant_action( + &tenant, + "Order", + entity_id, + "AddItem", + serde_json::json!({}), + &agent, + ) + .await + .expect("domain action dispatch must complete after source-fence retry"); + + assert!(updated.success, "domain action failed: {:?}", updated.error); + assert!(rewritten.load(Ordering::SeqCst)); + assert_eq!(append_attempts.load(Ordering::SeqCst), 2); + assert_eq!(updated.state.fields["WorkspaceId"], "ws-after-append"); + assert_eq!(updated.state.fields["Path"], "/journal-generation"); + assert_eq!(updated.state.item_count, 1); + assert_eq!( + EventStore::lookup_by_key( + &inner, + tenant.as_str(), + "Order", + "ws_path", + &ws_path_hash("ws-after-append", "/journal-generation"), + ) + .await + .expect("lookup recovered domain-action ownership") + .as_deref(), + Some(entity_id) + ); + assert_eq!( + EventStore::lookup_by_key( + &inner, + tenant.as_str(), + "Order", + "ws_path", + &ws_path_hash("ws-before-append", "/journal-generation"), + ) + .await + .expect("lookup stale domain-action ownership"), + None + ); +} diff --git a/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/contract_race.rs b/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/contract_race.rs new file mode 100644 index 000000000..37fc6e400 --- /dev/null +++ b/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/contract_race.rs @@ -0,0 +1,308 @@ +//! Forced interleaving proof for the key-contract read fence. + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use super::*; +use crate::storage::{BackendLabel, BoxedEventStore}; +use temper_runtime::persistence::{ + EntityKeyRow, EventMetadata, IndexReconciliation, PersistenceAppend, PersistenceAppendResult, + PersistenceEnvelope, PersistenceError, +}; +use temper_runtime::scheduler::{install_deterministic_context, sim_now, sim_uuid}; + +#[derive(Clone)] +struct ContractChangingLookupStore { + inner: SimEventStore, + persistence_id: String, + fired: Arc, +} + +impl EventStore for ContractChangingLookupStore { + fn supports_authoritative_key_index(&self) -> bool { + true + } + + async fn append( + &self, + persistence_id: &str, + expected_sequence: u64, + events: &[PersistenceEnvelope], + ) -> Result { + EventStore::append(&self.inner, persistence_id, expected_sequence, events).await + } + + async fn append_batch( + &self, + appends: &[PersistenceAppend], + ) -> Result, PersistenceError> { + EventStore::append_batch(&self.inner, appends).await + } + + async fn read_events( + &self, + persistence_id: &str, + from_sequence: u64, + ) -> Result, PersistenceError> { + EventStore::read_events(&self.inner, persistence_id, from_sequence).await + } + + async fn read_events_page( + &self, + persistence_id: &str, + from_sequence: u64, + through_sequence: u64, + limit: usize, + ) -> Result, PersistenceError> { + EventStore::read_events_page( + &self.inner, + persistence_id, + from_sequence, + through_sequence, + limit, + ) + .await + } + + async fn save_snapshot( + &self, + persistence_id: &str, + sequence_nr: u64, + snapshot: &[u8], + ) -> Result<(), PersistenceError> { + EventStore::save_snapshot(&self.inner, persistence_id, sequence_nr, snapshot).await + } + + async fn load_snapshot( + &self, + persistence_id: &str, + ) -> Result)>, PersistenceError> { + EventStore::load_snapshot(&self.inner, persistence_id).await + } + + async fn list_entity_ids( + &self, + tenant: &str, + ) -> Result, PersistenceError> { + EventStore::list_entity_ids(&self.inner, tenant).await + } + + async fn list_entity_ids_by_type( + &self, + tenant: &str, + entity_type: &str, + ) -> Result, PersistenceError> { + EventStore::list_entity_ids_by_type(&self.inner, tenant, entity_type).await + } + + async fn lookup_by_key( + &self, + tenant: &str, + entity_type: &str, + key_name: &str, + key_hash: &str, + ) -> Result, PersistenceError> { + if !self.fired.swap(true, Ordering::SeqCst) { + let sequence = EventStore::read_events(&self.inner, &self.persistence_id, 0) + .await? + .last() + .map(|event| event.sequence_nr) + .unwrap_or(0); + let timestamp = sim_now(); + let event = PersistenceEnvelope { + sequence_nr: 0, + event_type: "ContractBWrite".to_string(), + payload: serde_json::json!({ + "action": "ContractBWrite", + "from_status": "Draft", + "to_status": "Draft", + "timestamp": timestamp, + "params": {}, + "idempotency_key": null, + }), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: self.persistence_id.clone(), + }, + }; + EventStore::append_with_index_rows( + &self.inner, + &self.persistence_id, + sequence, + &[event], + &[], + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some("v3|10:contract_b[7:OtherId]".to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await?; + } + EventStore::lookup_by_key(&self.inner, tenant, entity_type, key_name, key_hash).await + } + + async fn key_index_backfilled_types( + &self, + tenant: &str, + ) -> Result, PersistenceError> { + EventStore::key_index_backfilled_types(&self.inner, tenant).await + } + + async fn key_index_reconciliation_revision( + &self, + tenant: &str, + entity_type: &str, + ) -> Result { + EventStore::key_index_reconciliation_revision(&self.inner, tenant, entity_type).await + } +} + +/// Coverage A is observed first; the lookup itself then commits a real exact-set +/// write under incompatible contract B. The post-lookup revision fence must force +/// a journal-backed scan, which still returns the entity matching the captured A +/// fields instead of treating B's key-index miss as authoritative absence. +#[tokio::test] +async fn incompatible_contract_write_between_coverage_and_lookup_falls_back_to_scan() { + let (_guard, _clock, _ids) = install_deterministic_context(246); + let tenant = TenantId::default(); + let entity_id = "ord-contract-race"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let workspace = "ws-race"; + let path = "/still-matches-a"; + let inner = SimEventStore::no_faults(246); + let timestamp = sim_now(); + let create = PersistenceEnvelope { + sequence_nr: 0, + event_type: "Create".to_string(), + payload: serde_json::json!({ + "action": "Create", + "from_status": "Draft", + "to_status": "Draft", + "timestamp": timestamp, + "params": {"WorkspaceId": workspace, "Path": path}, + "idempotency_key": null, + }), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: persistence_id.clone(), + }, + }; + EventStore::append_with_index_rows( + &inner, + &persistence_id, + 0, + &[create], + &[EntityKeyRow { + key_name: "ws_path".to_string(), + key_hash: ws_path_hash(workspace, path), + }], + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(ORDER_KEY_SET_SIGNATURE.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .expect("seed contract-A stream and ownership"); + EventStore::save_snapshot( + &inner, + &persistence_id, + 1, + &serde_json::to_vec(&serde_json::json!({ + "entity_type": "Order", + "entity_id": entity_id, + "status": "Draft", + "item_count": 0, + "fields": { + "Id": entity_id, + "Status": "Draft", + "WorkspaceId": workspace, + "Path": path, + }, + })) + .expect("snapshot serialization"), + ) + .await + .expect("seed replayable state"); + EventStore::mark_key_index_backfilled( + &inner, + tenant.as_str(), + "Order", + ORDER_KEY_SET_SIGNATURE, + ) + .await + .expect("mark contract-A coverage"); + + let fired = Arc::new(AtomicBool::new(false)); + let racing_store = ContractChangingLookupStore { + inner: inner.clone(), + persistence_id, + fired: fired.clone(), + }; + let mut state = build_order_state("query-plane-key-contract-race"); + state.set_storage_stack(StorageStack::new( + BackendLabel::Sim, + BoxedEventStore::new(racing_store), + None, + None, + None, + None, + None, + None, + None, + None, + )); + + let query_options = QueryOptions { + filter: Some(ws_path_filter(workspace, path)), + ..QueryOptions::default() + }; + let security_ctx = SecurityContext::system(); + let result = read_entity_set_page(QueryPlaneReadRequest { + state: &state, + tenant: &tenant, + security_ctx: &security_ctx, + entity_type: "Order", + entity_set_name: "Orders", + query_options: &query_options, + budget: QueryPlaneReadBudget { + default_page_size: 10, + max_entities: 10, + }, + }) + .await; + let result = match result { + Ok(result) => result, + Err(_) => panic!("revision change must fall back to authoritative scan"), + }; + + assert!( + fired.load(Ordering::SeqCst), + "forced lookup interleaving did not run" + ); + assert_eq!(result.entities.len(), 1); + assert_eq!( + result.entities[0]["entity_id"].as_str(), + Some(entity_id), + "scan result: {}", + result.entities[0] + ); + assert!( + EventStore::key_index_backfilled_types(&inner, tenant.as_str()) + .await + .expect("read invalidated coverage") + .is_empty(), + "contract-B write must invalidate contract-A coverage" + ); +} diff --git a/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/ownership_race.rs b/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/ownership_race.rs new file mode 100644 index 000000000..eeab5cb04 --- /dev/null +++ b/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/ownership_race.rs @@ -0,0 +1,455 @@ +//! Forced ownership/body-generation interleavings for exact-key reads. + +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + +use super::*; +use crate::storage::{BackendLabel, BoxedEventStore}; +use temper_runtime::persistence::{ + EntityKeyLookup, EntityKeyRow, EventMetadata, IndexReconciliation, PersistenceAppend, + PersistenceAppendResult, PersistenceEnvelope, PersistenceError, +}; +use temper_runtime::scheduler::{install_deterministic_context, sim_now, sim_uuid}; + +mod incomplete_coverage; + +fn field_update_event(persistence_id: &str, path: &str, token: &str) -> PersistenceEnvelope { + let timestamp = sim_now(); + PersistenceEnvelope { + sequence_nr: 0, + event_type: "Temper.Internal.FieldUpdate.v1".to_string(), + payload: serde_json::json!({ + "schema": "temper.field-update.v1", + "fields": {"Path": path}, + "replace": false, + "idempotency_key": token, + }), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: persistence_id.to_string(), + }, + } +} + +fn key_row(workspace: &str, path: &str) -> EntityKeyRow { + EntityKeyRow { + key_name: "ws_path".to_string(), + key_hash: ws_path_hash(workspace, path), + } +} + +fn snapshot(entity_id: &str, workspace: &str, path: &str) -> Vec { + serde_json::to_vec(&serde_json::json!({ + "entity_type": "Order", + "entity_id": entity_id, + "status": "Draft", + "item_count": 0, + "fields": { + "Id": entity_id, + "Status": "Draft", + "WorkspaceId": workspace, + "Path": path, + }, + })) + .expect("snapshot serialization") +} + +async fn seed_owner( + store: &SimEventStore, + tenant: &TenantId, + entity_id: &str, + workspace: &str, + path: &str, +) { + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let mut initial_event = field_update_event(&persistence_id, path, "seed-owner"); + initial_event.payload["fields"] = serde_json::json!({ + "Id": entity_id, + "WorkspaceId": workspace, + "Path": path, + }); + EventStore::append_with_index_rows( + store, + &persistence_id, + 0, + &[initial_event], + &[key_row(workspace, path)], + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(ORDER_KEY_SET_SIGNATURE.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .expect("seed owner journal and key row"); + EventStore::save_snapshot( + store, + &persistence_id, + 1, + &snapshot(entity_id, workspace, path), + ) + .await + .expect("seed owner snapshot"); +} + +#[tokio::test] +async fn stable_key_row_materializes_its_journal_generation_not_stale_actor() { + let (_guard, _clock, _ids) = install_deterministic_context(250); + let tenant = TenantId::default(); + let workspace = "ws-stale-actor"; + let stale_path = "/before"; + let current_path = "/after"; + let entity_id = "ord-stale-resident"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let (state, store) = build_order_state_with_sim("key-owner-stale-actor"); + + state + .get_or_create_tenant_entity( + &tenant, + "Order", + entity_id, + serde_json::json!({ + "Id": entity_id, + "WorkspaceId": workspace, + "Path": stale_path, + }), + ) + .await + .expect("spawn resident actor at sequence one"); + let sequence = current_sequence(&store, &tenant, "Order", entity_id).await; + EventStore::append_with_index_rows( + &store, + &persistence_id, + sequence, + &[field_update_event( + &persistence_id, + current_path, + "external-rename", + )], + &[key_row(workspace, current_path)], + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(ORDER_KEY_SET_SIGNATURE.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .expect("commit rename outside the resident actor"); + EventStore::mark_key_index_backfilled( + &store, + tenant.as_str(), + "Order", + ORDER_KEY_SET_SIGNATURE, + ) + .await + .expect("mark exact coverage"); + + let stale = state + .get_tenant_entity_state(&tenant, "Order", entity_id) + .await + .expect("resident actor remains readable"); + assert_eq!(stale.state.fields["Path"], stale_path); + + let options = QueryOptions { + filter: Some(ws_path_filter(workspace, current_path)), + ..QueryOptions::default() + }; + let security_ctx = SecurityContext::system(); + let result = read_entity_set_page(QueryPlaneReadRequest { + state: &state, + tenant: &tenant, + security_ctx: &security_ctx, + entity_type: "Order", + entity_set_name: "Orders", + query_options: &options, + budget: QueryPlaneReadBudget { + default_page_size: 10, + max_entities: 10, + }, + }) + .await; + let result = match result { + Ok(result) => result, + Err(_) => panic!("stable key row must materialize its journal generation"), + }; + + assert_eq!(result.entities.len(), 1); + assert_eq!(result.entities[0]["entity_id"], entity_id); + assert_eq!(result.entities[0]["fields"]["Path"], current_path); + assert_eq!(result.entities[0]["sequence_nr"], sequence + 1); +} + +#[derive(Clone)] +struct AbaTransferStore { + inner: SimEventStore, + tenant: String, + a_id: String, + b_id: String, + workspace: String, + target_path: String, + a_other_path: String, + b_other_path: String, + moved_once: Arc, + lookup_calls: Arc, +} + +impl AbaTransferStore { + fn a_persistence_id(&self) -> String { + format!("{}:Order:{}", self.tenant, self.a_id) + } + + fn b_persistence_id(&self) -> String { + format!("{}:Order:{}", self.tenant, self.b_id) + } + + async fn transfer(&self, restore_a: bool) -> Result<(), PersistenceError> { + let (a_path, b_path, expected, token) = if restore_a { + (&self.target_path, &self.b_other_path, 2, "aba-restore-a") + } else { + (&self.a_other_path, &self.target_path, 1, "aba-move-to-b") + }; + let a_pid = self.a_persistence_id(); + let b_pid = self.b_persistence_id(); + EventStore::append_batch( + &self.inner, + &[ + PersistenceAppend { + persistence_id: a_pid.clone(), + expected_sequence: expected, + events: vec![field_update_event(&a_pid, a_path, token)], + key_rows: vec![key_row(&self.workspace, a_path)], + reconcile_keys: true, + key_set_signature: Some(ORDER_KEY_SET_SIGNATURE.to_string()), + snapshot_source: Default::default(), + batch_idempotency: None, + }, + PersistenceAppend { + persistence_id: b_pid.clone(), + expected_sequence: expected, + events: vec![field_update_event(&b_pid, b_path, token)], + key_rows: vec![key_row(&self.workspace, b_path)], + reconcile_keys: true, + key_set_signature: Some(ORDER_KEY_SET_SIGNATURE.to_string()), + snapshot_source: Default::default(), + batch_idempotency: None, + }, + ], + ) + .await?; + Ok(()) + } +} + +impl EventStore for AbaTransferStore { + fn supports_authoritative_key_index(&self) -> bool { + true + } + + async fn append( + &self, + persistence_id: &str, + expected_sequence: u64, + events: &[PersistenceEnvelope], + ) -> Result { + EventStore::append(&self.inner, persistence_id, expected_sequence, events).await + } + + async fn append_batch( + &self, + appends: &[PersistenceAppend], + ) -> Result, PersistenceError> { + EventStore::append_batch(&self.inner, appends).await + } + + async fn read_events( + &self, + persistence_id: &str, + from_sequence: u64, + ) -> Result, PersistenceError> { + EventStore::read_events(&self.inner, persistence_id, from_sequence).await + } + + async fn read_events_page( + &self, + persistence_id: &str, + from_sequence: u64, + through_sequence: u64, + limit: usize, + ) -> Result, PersistenceError> { + if persistence_id == self.a_persistence_id() + && !self.moved_once.swap(true, Ordering::SeqCst) + { + self.transfer(false).await?; + } + EventStore::read_events_page( + &self.inner, + persistence_id, + from_sequence, + through_sequence, + limit, + ) + .await + } + + async fn save_snapshot( + &self, + persistence_id: &str, + sequence_nr: u64, + snapshot: &[u8], + ) -> Result<(), PersistenceError> { + EventStore::save_snapshot(&self.inner, persistence_id, sequence_nr, snapshot).await + } + + async fn load_snapshot( + &self, + persistence_id: &str, + ) -> Result)>, PersistenceError> { + EventStore::load_snapshot(&self.inner, persistence_id).await + } + + async fn list_entity_ids( + &self, + tenant: &str, + ) -> Result, PersistenceError> { + EventStore::list_entity_ids(&self.inner, tenant).await + } + + async fn list_entity_ids_by_type( + &self, + tenant: &str, + entity_type: &str, + ) -> Result, PersistenceError> { + EventStore::list_entity_ids_by_type(&self.inner, tenant, entity_type).await + } + + async fn lookup_by_key( + &self, + tenant: &str, + entity_type: &str, + key_name: &str, + key_hash: &str, + ) -> Result, PersistenceError> { + EventStore::lookup_by_key(&self.inner, tenant, entity_type, key_name, key_hash).await + } + + async fn lookup_by_key_with_sequence( + &self, + tenant: &str, + entity_type: &str, + key_name: &str, + key_hash: &str, + ) -> Result, PersistenceError> { + let call = self.lookup_calls.fetch_add(1, Ordering::SeqCst) + 1; + if call == 2 { + self.transfer(true).await?; + } + EventStore::lookup_by_key_with_sequence( + &self.inner, + tenant, + entity_type, + key_name, + key_hash, + ) + .await + } + + async fn key_index_backfilled_types( + &self, + tenant: &str, + ) -> Result, PersistenceError> { + EventStore::key_index_backfilled_types(&self.inner, tenant).await + } + + async fn key_index_reconciliation_revision( + &self, + tenant: &str, + entity_type: &str, + ) -> Result { + EventStore::key_index_reconciliation_revision(&self.inner, tenant, entity_type).await + } +} + +#[tokio::test] +async fn same_owner_aba_retries_the_new_journal_generation() { + let (_guard, _clock, _ids) = install_deterministic_context(251); + let tenant = TenantId::default(); + let workspace = "ws-aba"; + let target_path = "/target"; + let a_other_path = "/a-other"; + let b_other_path = "/b-other"; + let a_id = "ord-aba-a"; + let b_id = "ord-aba-b"; + let inner = SimEventStore::no_faults(251); + seed_owner(&inner, &tenant, a_id, workspace, target_path).await; + seed_owner(&inner, &tenant, b_id, workspace, b_other_path).await; + EventStore::mark_key_index_backfilled( + &inner, + tenant.as_str(), + "Order", + ORDER_KEY_SET_SIGNATURE, + ) + .await + .expect("mark exact coverage"); + + let lookup_calls = Arc::new(AtomicUsize::new(0)); + let racing = AbaTransferStore { + inner, + tenant: tenant.to_string(), + a_id: a_id.to_string(), + b_id: b_id.to_string(), + workspace: workspace.to_string(), + target_path: target_path.to_string(), + a_other_path: a_other_path.to_string(), + b_other_path: b_other_path.to_string(), + moved_once: Arc::new(AtomicBool::new(false)), + lookup_calls: lookup_calls.clone(), + }; + let mut state = build_order_state("key-owner-aba"); + state.set_storage_stack(StorageStack::new( + BackendLabel::Sim, + BoxedEventStore::new(racing), + None, + None, + None, + None, + None, + None, + None, + None, + )); + let options = QueryOptions { + filter: Some(ws_path_filter(workspace, target_path)), + ..QueryOptions::default() + }; + let security_ctx = SecurityContext::system(); + let result = read_entity_set_page(QueryPlaneReadRequest { + state: &state, + tenant: &tenant, + security_ctx: &security_ctx, + entity_type: "Order", + entity_set_name: "Orders", + query_options: &options, + budget: QueryPlaneReadBudget { + default_page_size: 10, + max_entities: 10, + }, + }) + .await; + let result = match result { + Ok(result) => result, + Err(_) => panic!("ABA must retry to a stable owner generation"), + }; + + assert_eq!(lookup_calls.load(Ordering::SeqCst), 3); + assert_eq!(result.entities.len(), 1); + assert_eq!(result.entities[0]["entity_id"], a_id); + assert_eq!(result.entities[0]["fields"]["Path"], target_path); + assert_eq!(result.entities[0]["sequence_nr"], 3); +} diff --git a/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/ownership_race/incomplete_coverage.rs b/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/ownership_race/incomplete_coverage.rs new file mode 100644 index 000000000..d30caa5b9 --- /dev/null +++ b/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/ownership_race/incomplete_coverage.rs @@ -0,0 +1,118 @@ +use super::*; + +mod catalog_fault; +mod composite_events; +mod fault_paths; +mod projection_race; +mod source_transitions; + +#[tokio::test] +async fn incomplete_key_scan_replays_journal_not_stale_resident_actor() { + let (_guard, _clock, _ids) = install_deterministic_context(249); + let tenant = TenantId::default(); + let workspace = "ws-incomplete-stale-actor"; + let stale_path = "/before"; + let current_path = "/after"; + let entity_id = "ord-incomplete-stale-resident"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let (state, store) = build_order_state_with_sim("incomplete-key-stale-actor"); + + state + .get_or_create_tenant_entity( + &tenant, + "Order", + entity_id, + serde_json::json!({ + "Id": entity_id, + "WorkspaceId": workspace, + "Path": stale_path, + }), + ) + .await + .expect("spawn resident actor at sequence one"); + let sequence = current_sequence(&store, &tenant, "Order", entity_id).await; + EventStore::append_with_index_rows( + &store, + &persistence_id, + sequence, + &[field_update_event( + &persistence_id, + current_path, + "external-rename-before-coverage", + )], + &[key_row(workspace, current_path)], + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(ORDER_KEY_SET_SIGNATURE.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .expect("commit rename outside the resident actor"); + + assert!( + EventStore::key_index_backfilled_types(&store, tenant.as_str()) + .await + .expect("read incomplete coverage") + .is_empty(), + "the read must take the incomplete-coverage authoritative scan" + ); + let stale = state + .get_tenant_entity_state(&tenant, "Order", entity_id) + .await + .expect("resident actor remains readable"); + assert_eq!(stale.state.fields["Path"], stale_path); + + let security_ctx = SecurityContext::system(); + let read_path = |path: &str| QueryOptions { + filter: Some(ws_path_filter(workspace, path)), + ..QueryOptions::default() + }; + let old_options = read_path(stale_path); + let old_result = read_entity_set_page(QueryPlaneReadRequest { + state: &state, + tenant: &tenant, + security_ctx: &security_ctx, + entity_type: "Order", + entity_set_name: "Orders", + query_options: &old_options, + budget: QueryPlaneReadBudget { + default_page_size: 10, + max_entities: 10, + }, + }) + .await; + let old_result = match old_result { + Ok(result) => result, + Err(_) => panic!("old-key authoritative scan must remain available"), + }; + assert!( + old_result.entities.is_empty(), + "an incomplete-key scan must not return the stale resident actor at sequence {sequence}" + ); + + let current_options = read_path(current_path); + let current_result = read_entity_set_page(QueryPlaneReadRequest { + state: &state, + tenant: &tenant, + security_ctx: &security_ctx, + entity_type: "Order", + entity_set_name: "Orders", + query_options: ¤t_options, + budget: QueryPlaneReadBudget { + default_page_size: 10, + max_entities: 10, + }, + }) + .await; + let current_result = match current_result { + Ok(result) => result, + Err(_) => panic!("current-key authoritative scan must remain available"), + }; + assert_eq!(current_result.entities.len(), 1); + assert_eq!(current_result.entities[0]["entity_id"], entity_id); + assert_eq!(current_result.entities[0]["fields"]["Path"], current_path); + assert_eq!(current_result.entities[0]["sequence_nr"], sequence + 1); +} diff --git a/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/ownership_race/incomplete_coverage/catalog_fault.rs b/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/ownership_race/incomplete_coverage/catalog_fault.rs new file mode 100644 index 000000000..e3e2d09b2 --- /dev/null +++ b/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/ownership_race/incomplete_coverage/catalog_fault.rs @@ -0,0 +1,456 @@ +use std::sync::Arc; + +use async_trait::async_trait; + +use super::*; +use crate::odata::read_support::{ + AuthoritativeMaterializationError, CatalogMaterializationPolicy, + materialize_entity_set_entities, +}; +use crate::storage::{EntityCatalogRow, QueryPlaneStore, QueryProjectionFieldsRow, StorageStack}; +use temper_runtime::persistence::ProjectionSourceFence; + +struct CatalogReadFault { + durable_row: Option, + fail_reads: bool, +} + +#[async_trait] +impl QueryPlaneStore for CatalogReadFault { + async fn upsert_projection( + &self, + _tenant: &str, + _entity_type: &str, + _entity_id: &str, + _status: &str, + _fields: &serde_json::Value, + _state: &serde_json::Value, + _sequence_nr: u64, + ) -> Result<(), PersistenceError> { + Ok(()) + } + + async fn upsert_projection_if_source( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + status: &str, + fields: &serde_json::Value, + state: &serde_json::Value, + sequence_nr: u64, + _source: ProjectionSourceFence<'_>, + ) -> Result { + self.upsert_projection( + tenant, + entity_type, + entity_id, + status, + fields, + state, + sequence_nr, + ) + .await?; + Ok(true) + } + + async fn remove_projection( + &self, + _tenant: &str, + _entity_type: &str, + _entity_id: &str, + ) -> Result<(), PersistenceError> { + Ok(()) + } + + async fn remove_projection_if_source( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + _source: ProjectionSourceFence<'_>, + ) -> Result { + self.remove_projection(tenant, entity_type, entity_id) + .await?; + Ok(true) + } + + async fn clear_projection_dirty_if_source( + &self, + _tenant: &str, + _entity_type: &str, + _entity_id: &str, + _source: ProjectionSourceFence<'_>, + ) -> Result { + Ok(true) + } + + async fn remove_projection_if_exact( + &self, + _tenant: &str, + _entity_type: &str, + _entity_id: &str, + _status: &str, + _fields: &serde_json::Value, + _state: &serde_json::Value, + _sequence_nr: u64, + ) -> Result { + Ok(false) + } + + async fn query_field_index( + &self, + _tenant: &str, + _entity_type: &str, + _where_clause: &str, + _params: Vec, + ) -> Result>, PersistenceError> { + Ok(None) + } + + async fn load_projection_fields_many( + &self, + _tenant: &str, + _entity_type: &str, + _entity_ids: &[String], + _field_names: &[&str], + ) -> Result>, PersistenceError> { + Ok(None) + } + + async fn load_entity_catalog_rows( + &self, + _tenant: &str, + _entity_type: &str, + entity_ids: &[String], + ) -> Result>, PersistenceError> { + if let Some(row) = self.durable_row.as_ref() { + assert!( + entity_ids.contains(&row.entity_id), + "the durable catalog-only candidate must be requested" + ); + } + if self.fail_reads { + return Err(PersistenceError::Storage( + "injected catalog read fault".to_string(), + )); + } + Ok(Some(self.durable_row.clone().into_iter().collect())) + } + + async fn projected_entity_counts_by_tenant( + &self, + ) -> Result>, PersistenceError> { + Ok(None) + } +} + +#[tokio::test] +async fn catalog_fault_cannot_turn_a_catalog_only_candidate_into_actor_state() { + let (_guard, _clock, _ids) = install_deterministic_context(264); + let tenant = TenantId::default(); + let entity_id = "ord-catalog-read-fault"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let events = SimEventStore::no_faults(264); + let query_plane = Arc::new(CatalogReadFault { + durable_row: Some(EntityCatalogRow { + entity_id: entity_id.to_string(), + status: "Draft".to_string(), + fields: serde_json::json!({ + "Id": entity_id, + "WorkspaceId": "ws-catalog-fault", + "Path": "/catalog-only", + }), + state: None, + sequence_nr: 1, + }), + fail_reads: true, + }); + let mut state = build_order_state("catalog-only-read-fault"); + state.set_storage_stack(StorageStack::new( + BackendLabel::Sim, + BoxedEventStore::new(events.clone()), + None, + None, + None, + None, + Some(query_plane), + None, + None, + None, + )); + + let materialized = materialize_entity_set_entities( + &state, + &tenant, + "Order", + "Orders", + &[entity_id.to_string()], + CatalogMaterializationPolicy::JournalAbsentOnly, + None, + ) + .await; + + assert!(materialized.entities.is_empty()); + assert_eq!( + materialized.error, + Some(AuthoritativeMaterializationError::JournalUnstable), + "a catalog read failure must remain retryable, not fall through to a fabricated actor body" + ); + assert_eq!( + EventStore::journal_boundary(&events, &persistence_id) + .await + .expect("journal remains readable") + .latest_sequence, + 0, + "the compatibility fallback must not write a first journal generation" + ); +} + +#[tokio::test] +async fn catalog_fault_does_not_disable_journal_backed_materialization() { + let (_guard, _clock, _ids) = install_deterministic_context(266); + let tenant = TenantId::default(); + let workspace = "ws-journal-with-catalog-fault"; + let journal_path = "/journal-authority"; + let entity_id = "ord-journal-with-catalog-fault"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let events = SimEventStore::no_faults(266); + + EventStore::append_with_index_rows( + &events, + &persistence_id, + 0, + &[super::source_transitions::complete_field_update( + &persistence_id, + entity_id, + workspace, + journal_path, + "journal-with-catalog-fault", + )], + &[key_row(workspace, journal_path)], + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(ORDER_KEY_SET_SIGNATURE.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .expect("seed authoritative journal generation"); + + let query_plane = Arc::new(CatalogReadFault { + durable_row: Some(EntityCatalogRow { + entity_id: entity_id.to_string(), + status: "Draft".to_string(), + fields: serde_json::json!({ + "Id": entity_id, + "WorkspaceId": workspace, + "Path": "/stale-catalog", + }), + state: None, + sequence_nr: 1, + }), + fail_reads: true, + }); + let mut state = build_order_state("journal-backed-catalog-fault"); + state.set_storage_stack(StorageStack::new( + BackendLabel::Sim, + BoxedEventStore::new(events), + None, + None, + None, + None, + Some(query_plane), + None, + None, + None, + )); + + let materialized = materialize_entity_set_entities( + &state, + &tenant, + "Order", + "Orders", + &[entity_id.to_string()], + CatalogMaterializationPolicy::JournalAbsentOnly, + None, + ) + .await; + + assert_eq!(materialized.error, None); + assert_eq!(materialized.entities.len(), 1); + assert_eq!(materialized.entities[0]["fields"]["Path"], journal_path); +} + +#[tokio::test] +async fn empty_catalog_does_not_bootstrap_a_projection_only_candidate() { + let (_guard, _clock, _ids) = install_deterministic_context(270); + let tenant = TenantId::default(); + let entity_id = "ord-projection-only-ghost"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let events = SimEventStore::no_faults(270); + let query_plane = Arc::new(CatalogReadFault { + durable_row: None, + fail_reads: false, + }); + let mut state = build_order_state("projection-only-catalog-miss"); + state.set_storage_stack(StorageStack::new( + BackendLabel::Sim, + BoxedEventStore::new(events.clone()), + None, + None, + None, + None, + Some(query_plane), + None, + None, + None, + )); + + let materialized = materialize_entity_set_entities( + &state, + &tenant, + "Order", + "Orders", + &[entity_id.to_string()], + CatalogMaterializationPolicy::JournalAbsentOnly, + None, + ) + .await; + + assert!(materialized.entities.is_empty()); + assert_eq!(materialized.error, None); + assert_eq!( + EventStore::journal_boundary(&events, &persistence_id) + .await + .expect("journal remains readable") + .latest_sequence, + 0, + "a read must not turn a projection-only candidate into durable actor state" + ); +} + +#[tokio::test] +async fn snapshot_only_source_outranks_stale_catalog_compatibility() { + let (_guard, _clock, _ids) = install_deterministic_context(271); + let tenant = TenantId::default(); + let entity_id = "ord-snapshot-over-catalog"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let events = SimEventStore::no_faults(271); + EventStore::save_snapshot( + &events, + &persistence_id, + 2, + &snapshot(entity_id, "ws-snapshot", "/snapshot-authority"), + ) + .await + .expect("seed snapshot-only durable source"); + let query_plane = Arc::new(CatalogReadFault { + durable_row: Some(EntityCatalogRow { + entity_id: entity_id.to_string(), + status: "Draft".to_string(), + fields: serde_json::json!({ + "Id": entity_id, + "WorkspaceId": "ws-catalog", + "Path": "/stale-catalog", + }), + state: None, + sequence_nr: 3, + }), + fail_reads: false, + }); + let mut state = build_order_state("snapshot-over-stale-catalog"); + state.set_storage_stack(StorageStack::new( + BackendLabel::Sim, + BoxedEventStore::new(events), + None, + None, + None, + None, + Some(query_plane), + None, + None, + None, + )); + + let materialized = materialize_entity_set_entities( + &state, + &tenant, + "Order", + "Orders", + &[entity_id.to_string()], + CatalogMaterializationPolicy::JournalAbsentOnly, + None, + ) + .await; + + assert_eq!(materialized.error, None); + assert_eq!(materialized.entities.len(), 1); + assert_eq!( + materialized.entities[0]["fields"]["WorkspaceId"], + "ws-snapshot" + ); + assert_eq!( + materialized.entities[0]["fields"]["Path"], + "/snapshot-authority" + ); +} + +#[tokio::test] +async fn incompatible_snapshot_cannot_be_masked_by_catalog_compatibility() { + let (_guard, _clock, _ids) = install_deterministic_context(272); + let tenant = TenantId::default(); + let entity_id = "ord-invalid-snapshot-catalog"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let events = SimEventStore::no_faults(272); + EventStore::save_snapshot(&events, &persistence_id, 1, b"not a valid entity snapshot") + .await + .expect("seed incompatible snapshot source"); + let query_plane = Arc::new(CatalogReadFault { + durable_row: Some(EntityCatalogRow { + entity_id: entity_id.to_string(), + status: "Draft".to_string(), + fields: serde_json::json!({ + "Id": entity_id, + "WorkspaceId": "ws-catalog", + "Path": "/catalog-must-not-mask-snapshot", + }), + state: None, + sequence_nr: 1, + }), + fail_reads: false, + }); + let mut state = build_order_state("invalid-snapshot-catalog-mask"); + state.set_storage_stack(StorageStack::new( + BackendLabel::Sim, + BoxedEventStore::new(events), + None, + None, + None, + None, + Some(query_plane), + None, + None, + None, + )); + + let materialized = materialize_entity_set_entities( + &state, + &tenant, + "Order", + "Orders", + &[entity_id.to_string()], + CatalogMaterializationPolicy::JournalAbsentOnly, + None, + ) + .await; + + assert!(materialized.entities.is_empty()); + assert_eq!( + materialized.error, + Some(AuthoritativeMaterializationError::JournalUnstable), + "an incompatible snapshot must fail closed instead of serving stale catalog state" + ); +} diff --git a/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/ownership_race/incomplete_coverage/composite_events.rs b/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/ownership_race/incomplete_coverage/composite_events.rs new file mode 100644 index 000000000..551a91bcb --- /dev/null +++ b/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/ownership_race/incomplete_coverage/composite_events.rs @@ -0,0 +1,124 @@ +use super::*; +use crate::entity_actor::EntityEvent; +use temper_runtime::persistence::COMPOSITE_EVENT_TYPE; + +fn composite_named_domain_event( + persistence_id: &str, + entity_id: &str, + workspace: &str, + path: &str, +) -> PersistenceEnvelope { + let timestamp = sim_now(); + let event = EntityEvent { + action: COMPOSITE_EVENT_TYPE.to_string(), + from_status: "Draft".to_string(), + to_status: "Draft".to_string(), + timestamp, + params: serde_json::json!({ + "Id": entity_id, + "WorkspaceId": workspace, + "Path": path, + }), + idempotency_key: Some("domain-composite-name".to_string()), + }; + PersistenceEnvelope { + sequence_nr: 0, + event_type: COMPOSITE_EVENT_TYPE.to_string(), + payload: serde_json::to_value(event).expect("serialize domain event"), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: persistence_id.to_string(), + }, + } +} + +#[tokio::test] +async fn composite_event_type_only_skips_the_runtime_audit_schema() { + let (_guard, _clock, _ids) = install_deterministic_context(262); + let tenant = TenantId::default(); + let workspace = "ws-domain-composite"; + let path = "/domain-action"; + let entity_id = "ord-domain-composite"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let (state, store) = build_order_state_with_sim("domain-composite-event-name"); + + EventStore::append_with_index_rows( + &store, + &persistence_id, + 0, + &[composite_named_domain_event( + &persistence_id, + entity_id, + workspace, + path, + )], + &[key_row(workspace, path)], + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(ORDER_KEY_SET_SIGNATURE.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .expect("seed domain event whose legal action name matches the audit record type"); + + let result = match super::source_transitions::read_path(&state, &tenant, workspace, path).await + { + Ok(result) => result, + Err(_) => panic!("domain event must remain replayable"), + }; + assert_eq!(result.entities.len(), 1); + assert_eq!(result.entities[0]["entity_id"], entity_id); + assert_eq!(result.entities[0]["fields"]["Path"], path); +} + +#[tokio::test] +async fn malformed_composite_audit_record_is_not_silently_authoritative() { + let (_guard, _clock, _ids) = install_deterministic_context(263); + let tenant = TenantId::default(); + let workspace = "ws-malformed-composite"; + let path = "/malformed-audit"; + let entity_id = "ord-malformed-composite"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let (state, store) = build_order_state_with_sim("malformed-composite-audit"); + let timestamp = sim_now(); + + EventStore::append_with_index_rows( + &store, + &persistence_id, + 0, + &[PersistenceEnvelope { + sequence_nr: 0, + event_type: COMPOSITE_EVENT_TYPE.to_string(), + payload: serde_json::json!({"corrupt": true}), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: persistence_id.clone(), + }, + }], + &[key_row(workspace, path)], + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(ORDER_KEY_SET_SIGNATURE.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .expect("seed malformed composite audit record"); + + let result = super::source_transitions::read_path(&state, &tenant, workspace, path).await; + assert!( + matches!(result, Err(QueryPlaneReadError::KeyOwnershipUnstable)), + "strict authoritative replay must reject an undecodable composite record" + ); +} diff --git a/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/ownership_race/incomplete_coverage/fault_paths.rs b/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/ownership_race/incomplete_coverage/fault_paths.rs new file mode 100644 index 000000000..74903719f --- /dev/null +++ b/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/ownership_race/incomplete_coverage/fault_paths.rs @@ -0,0 +1,350 @@ +use super::*; +use crate::entity_actor::{EntityEvent, EntityRecoveryContext, recover_entity_state_from_store}; + +async fn read_path( + state: &ServerState, + tenant: &TenantId, + workspace: &str, + path: &str, +) -> Result { + let options = QueryOptions { + filter: Some(ws_path_filter(workspace, path)), + ..QueryOptions::default() + }; + let security_ctx = SecurityContext::system(); + read_entity_set_page(QueryPlaneReadRequest { + state, + tenant, + security_ctx: &security_ctx, + entity_type: "Order", + entity_set_name: "Orders", + query_options: &options, + budget: QueryPlaneReadBudget { + default_page_size: 10, + max_entities: 10, + }, + }) + .await +} + +fn expect_read( + result: Result, + message: &str, +) -> QueryPlaneReadResult { + match result { + Ok(result) => result, + Err(_) => panic!("{message}"), + } +} + +fn tombstone_event(persistence_id: &str) -> PersistenceEnvelope { + let timestamp = sim_now(); + let event = EntityEvent { + action: "Delete".to_string(), + from_status: "Draft".to_string(), + to_status: "Deleted".to_string(), + timestamp, + params: serde_json::json!({}), + idempotency_key: Some("terminal-delete".to_string()), + }; + PersistenceEnvelope { + sequence_nr: 0, + event_type: "Delete".to_string(), + payload: serde_json::to_value(event).expect("serialize tombstone"), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: persistence_id.to_string(), + }, + } +} + +fn complete_field_update( + persistence_id: &str, + entity_id: &str, + workspace: &str, + path: &str, + token: &str, +) -> PersistenceEnvelope { + let mut event = field_update_event(persistence_id, path, token); + event.payload["fields"] = serde_json::json!({ + "Id": entity_id, + "WorkspaceId": workspace, + "Path": path, + }); + event +} + +#[tokio::test] +async fn exhausted_journal_faults_return_unstable_not_authoritative_absence() { + let (_guard, _clock, _ids) = install_deterministic_context(255); + let tenant = TenantId::default(); + let workspace = "ws-replay-fault"; + let path = "/present"; + let entity_id = "ord-replay-fault"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let (state, store) = build_order_state_with_sim("incomplete-replay-fault"); + + EventStore::append_with_index_rows( + &store, + &persistence_id, + 0, + &[field_update_event( + &persistence_id, + path, + "faulted-present-entity", + )], + &[key_row(workspace, path)], + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(ORDER_KEY_SET_SIGNATURE.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .expect("seed present journal entity"); + store.fail_next_reads(&persistence_id, 3); + + let result = read_path(&state, &tenant, workspace, path).await; + assert!( + matches!(result, Err(QueryPlaneReadError::KeyOwnershipUnstable)), + "journal uncertainty must not be flattened into an empty successful read" + ); +} + +#[tokio::test] +async fn incompatible_journal_event_returns_unstable_not_authoritative_absence() { + let (_guard, _clock, _ids) = install_deterministic_context(259); + let tenant = TenantId::default(); + let workspace = "ws-incompatible-journal"; + let path = "/durably-present"; + let entity_id = "ord-incompatible-journal"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let (state, store) = build_order_state_with_sim("incomplete-incompatible-journal"); + let timestamp = sim_now(); + EventStore::append_with_index_rows( + &store, + &persistence_id, + 0, + &[PersistenceEnvelope { + sequence_nr: 0, + event_type: "LegacyCreate".to_string(), + payload: serde_json::json!({"legacy_schema": true}), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: persistence_id.clone(), + }, + }], + &[key_row(workspace, path)], + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(ORDER_KEY_SET_SIGNATURE.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .expect("seed incompatible durable journal event"); + + let result = read_path(&state, &tenant, workspace, path).await; + assert!( + matches!(result, Err(QueryPlaneReadError::KeyOwnershipUnstable)), + "an undecodable authoritative event must not become a successful empty read" + ); +} + +#[tokio::test] +async fn journal_only_materialization_pages_beyond_snapshot_tail_budget() { + let (_guard, _clock, _ids) = install_deterministic_context(260); + let tenant = TenantId::default(); + let workspace = "ws-long-lived-journal"; + let snapshot_path = "/at-snapshot"; + let final_path = "/after-snapshot"; + let entity_id = "ord-long-lived-journal"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let (state, store) = build_order_state_with_sim("incomplete-long-lived-journal"); + let event_count = crate::entity_actor::types::MAX_EVENTS_SINCE_SNAPSHOT + 1; + let mut events = Vec::with_capacity(event_count); + events.push(complete_field_update( + &persistence_id, + entity_id, + workspace, + "/initial", + "long-lived-0", + )); + for index in 1..event_count { + let path = if index + 1 == event_count { + final_path + } else if index + 1 == event_count - 1 { + snapshot_path + } else { + "/intermediate" + }; + events.push(field_update_event( + &persistence_id, + path, + &format!("long-lived-{index}"), + )); + } + EventStore::append_with_index_rows( + &store, + &persistence_id, + 0, + &events, + &[key_row(workspace, final_path)], + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(ORDER_KEY_SET_SIGNATURE.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .expect("seed long-lived journal"); + EventStore::save_snapshot( + &store, + &persistence_id, + (event_count - 1) as u64, + &snapshot(entity_id, workspace, snapshot_path), + ) + .await + .expect("seed valid near-head snapshot"); + + let current = expect_read( + read_path(&state, &tenant, workspace, final_path).await, + "long-lived journal must page to the current generation", + ); + assert_eq!(current.entities.len(), 1); + assert_eq!(current.entities[0]["fields"]["Path"], final_path); + assert_eq!(current.entities[0]["sequence_nr"], event_count as u64); +} + +#[tokio::test] +async fn incomplete_scan_keeps_tombstone_terminal_through_legacy_suffix() { + let (_guard, _clock, _ids) = install_deterministic_context(256); + let tenant = TenantId::default(); + let workspace = "ws-terminal-suffix"; + let path = "/deleted"; + let entity_id = "ord-terminal-suffix"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let (state, store) = build_order_state_with_sim("incomplete-terminal-suffix"); + + EventStore::append_with_index_rows( + &store, + &persistence_id, + 0, + &[field_update_event( + &persistence_id, + path, + "terminal-live-generation", + )], + &[key_row(workspace, path)], + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(ORDER_KEY_SET_SIGNATURE.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .expect("seed live journal state"); + EventStore::append_with_index_rows( + &store, + &persistence_id, + 1, + &[tombstone_event(&persistence_id)], + &[], + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(ORDER_KEY_SET_SIGNATURE.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .expect("append terminal tombstone"); + EventStore::append_with_index_rows( + &store, + &persistence_id, + 2, + &[field_update_event( + &persistence_id, + "/legacy-resurrection", + "legacy-suffix", + )], + &[], + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(ORDER_KEY_SET_SIGNATURE.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .expect("append legacy suffix after tombstone"); + EventStore::save_snapshot( + &store, + &persistence_id, + 3, + &snapshot(entity_id, workspace, path), + ) + .await + .expect("seed newer stale live snapshot"); + + let result = expect_read( + read_path(&state, &tenant, workspace, path).await, + "terminal suffix scan must remain available", + ); + assert!( + result.entities.is_empty(), + "the terminal tombstone must win and the full suffix high-water must be consumed" + ); + let suffix = expect_read( + read_path(&state, &tenant, workspace, "/legacy-resurrection").await, + "legacy-suffix key scan must remain available", + ); + assert!( + suffix.entities.is_empty(), + "a post-tombstone suffix must not resurrect ownership under its own key" + ); + + let table = { + let registry = state.registry.read().expect("registry lock poisoned"); + registry + .get_table_live(&tenant, "Order") + .expect("Order transition table") + .read() + .expect("table lock poisoned") + .clone() + }; + let (journal, backend) = state.event_journal().expect("sim event journal"); + let recovered = recover_entity_state_from_store( + EntityRecoveryContext { + tenant: tenant.as_str(), + entity_type: "Order", + entity_id, + table: &table, + store: &journal, + backend, + initial_fields: &serde_json::json!({}), + blob_store: None, + }, + true, + ) + .await + .expect("strict terminal recovery"); + assert_eq!(recovered.status, "Deleted"); + assert_eq!(recovered.sequence_nr, 3); +} diff --git a/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/ownership_race/incomplete_coverage/projection_race.rs b/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/ownership_race/incomplete_coverage/projection_race.rs new file mode 100644 index 000000000..a05320002 --- /dev/null +++ b/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/ownership_race/incomplete_coverage/projection_race.rs @@ -0,0 +1,342 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; + +use async_trait::async_trait; + +use super::*; +use crate::entity_actor::EntityEvent; +use crate::storage::{ + BackendLabel, BoxedEventStore, EntityCatalogRow, QueryPlaneStore, QueryProjectionFieldsRow, +}; +use temper_runtime::persistence::{PersistenceError, ProjectionSourceFence}; + +#[derive(Clone)] +struct TombstoneOnUpsertQueryPlane { + inner: Arc, + events: SimEventStore, + persistence_id: String, + fired: Arc, +} + +impl TombstoneOnUpsertQueryPlane { + fn tombstone(&self) -> PersistenceEnvelope { + let timestamp = sim_now(); + let event = EntityEvent { + action: "Delete".to_string(), + from_status: "Draft".to_string(), + to_status: "Deleted".to_string(), + timestamp, + params: serde_json::json!({}), + idempotency_key: Some("projection-race-delete".to_string()), + }; + PersistenceEnvelope { + sequence_nr: 0, + event_type: "Delete".to_string(), + payload: serde_json::to_value(event).expect("serialize projection-race tombstone"), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: self.persistence_id.clone(), + }, + } + } + + async fn source_matches( + &self, + source: ProjectionSourceFence<'_>, + ) -> Result { + let boundary = EventStore::journal_boundary(&self.events, &self.persistence_id).await?; + if boundary.latest_sequence != source.expected_journal_sequence { + return Ok(false); + } + let snapshot = EventStore::load_snapshot(&self.events, &self.persistence_id).await?; + Ok(match (source.expected_snapshot, snapshot.as_ref()) { + (None, None) => true, + (Some(expected), Some((sequence_nr, state))) => { + expected.sequence_nr == *sequence_nr && expected.state == state.as_slice() + } + _ => false, + }) + } +} + +#[async_trait] +impl QueryPlaneStore for TombstoneOnUpsertQueryPlane { + async fn upsert_projection( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + status: &str, + fields: &serde_json::Value, + state: &serde_json::Value, + sequence_nr: u64, + ) -> Result<(), PersistenceError> { + if !self.fired.swap(true, Ordering::SeqCst) { + EventStore::append_with_index_rows( + &self.events, + &self.persistence_id, + sequence_nr, + &[self.tombstone()], + &[], + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(ORDER_KEY_SET_SIGNATURE.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await?; + QueryPlaneStore::remove_projection(self.inner.as_ref(), tenant, entity_type, entity_id) + .await?; + } + QueryPlaneStore::upsert_projection( + self.inner.as_ref(), + tenant, + entity_type, + entity_id, + status, + fields, + state, + sequence_nr, + ) + .await + } + + async fn upsert_projection_if_source( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + status: &str, + fields: &serde_json::Value, + state: &serde_json::Value, + sequence_nr: u64, + source: ProjectionSourceFence<'_>, + ) -> Result { + if !self.fired.swap(true, Ordering::SeqCst) { + EventStore::append_with_index_rows( + &self.events, + &self.persistence_id, + source.expected_journal_sequence, + &[self.tombstone()], + &[], + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(ORDER_KEY_SET_SIGNATURE.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await?; + QueryPlaneStore::remove_projection(self.inner.as_ref(), tenant, entity_type, entity_id) + .await?; + } + if !self.source_matches(source).await? { + return Ok(false); + } + QueryPlaneStore::upsert_projection( + self.inner.as_ref(), + tenant, + entity_type, + entity_id, + status, + fields, + state, + sequence_nr, + ) + .await?; + Ok(true) + } + + async fn remove_projection( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + ) -> Result<(), PersistenceError> { + QueryPlaneStore::remove_projection(self.inner.as_ref(), tenant, entity_type, entity_id) + .await + } + + async fn remove_projection_if_source( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + source: ProjectionSourceFence<'_>, + ) -> Result { + if !self.source_matches(source).await? { + return Ok(false); + } + QueryPlaneStore::remove_projection(self.inner.as_ref(), tenant, entity_type, entity_id) + .await?; + Ok(true) + } + + async fn remove_projection_if_exact( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + status: &str, + fields: &serde_json::Value, + state: &serde_json::Value, + sequence_nr: u64, + ) -> Result { + let rows = QueryPlaneStore::load_entity_catalog_rows( + self.inner.as_ref(), + tenant, + entity_type, + &[entity_id.to_string()], + ) + .await? + .unwrap_or_default(); + let exact = rows.first().is_some_and(|row| { + row.status == status + && &row.fields == fields + && row.state.as_ref() == Some(state) + && row.sequence_nr == sequence_nr + }); + if exact { + QueryPlaneStore::remove_projection(self.inner.as_ref(), tenant, entity_type, entity_id) + .await?; + } + Ok(exact) + } + + async fn query_field_index( + &self, + tenant: &str, + entity_type: &str, + where_clause: &str, + params: Vec, + ) -> Result>, PersistenceError> { + QueryPlaneStore::query_field_index( + self.inner.as_ref(), + tenant, + entity_type, + where_clause, + params, + ) + .await + } + + async fn load_projection_fields_many( + &self, + tenant: &str, + entity_type: &str, + entity_ids: &[String], + field_names: &[&str], + ) -> Result>, PersistenceError> { + QueryPlaneStore::load_projection_fields_many( + self.inner.as_ref(), + tenant, + entity_type, + entity_ids, + field_names, + ) + .await + } + + async fn load_entity_catalog_rows( + &self, + tenant: &str, + entity_type: &str, + entity_ids: &[String], + ) -> Result>, PersistenceError> { + QueryPlaneStore::load_entity_catalog_rows( + self.inner.as_ref(), + tenant, + entity_type, + entity_ids, + ) + .await + } + + async fn projected_entity_counts_by_tenant( + &self, + ) -> Result>, PersistenceError> { + QueryPlaneStore::projected_entity_counts_by_tenant(self.inner.as_ref()).await + } +} + +#[tokio::test] +async fn tombstone_between_replay_and_projection_repair_is_replayed_before_return() { + let (_guard, _clock, _ids) = install_deterministic_context(261); + let tenant = TenantId::default(); + let workspace = "ws-projection-repair-race"; + let path = "/live-before-repair"; + let entity_id = "ord-projection-repair-race"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let events = SimEventStore::no_faults(261); + EventStore::append_with_index_rows( + &events, + &persistence_id, + 0, + &[super::source_transitions::complete_field_update( + &persistence_id, + entity_id, + workspace, + path, + "projection-race-live", + )], + &[key_row(workspace, path)], + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(ORDER_KEY_SET_SIGNATURE.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .expect("seed projection-race live state"); + let catalog = Arc::new(SimQueryPlane::default()); + let racing_query_plane = Arc::new(TombstoneOnUpsertQueryPlane { + inner: catalog.clone(), + events: events.clone(), + persistence_id, + fired: Arc::new(AtomicBool::new(false)), + }); + let mut state = build_order_state("projection-repair-tombstone-race"); + state.set_storage_stack(StorageStack::new( + BackendLabel::Sim, + BoxedEventStore::new(events), + None, + None, + None, + None, + Some(racing_query_plane), + None, + None, + None, + )); + + let result = match super::source_transitions::read_path(&state, &tenant, workspace, path).await + { + Ok(result) => result, + Err(_) => panic!("a tombstone racing repair must stabilize to a successful read"), + }; + assert!( + result.entities.is_empty(), + "a tombstone racing repair must not return the stale live body" + ); + let rows = QueryPlaneStore::load_entity_catalog_rows( + catalog.as_ref(), + tenant.as_str(), + "Order", + &[entity_id.to_string()], + ) + .await + .expect("load projection after repair race") + .expect("sim catalog support"); + assert!( + rows.is_empty(), + "terminal retry must remove stale projection" + ); +} diff --git a/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/ownership_race/incomplete_coverage/source_transitions.rs b/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/ownership_race/incomplete_coverage/source_transitions.rs new file mode 100644 index 000000000..26b3e6d01 --- /dev/null +++ b/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/ownership_race/incomplete_coverage/source_transitions.rs @@ -0,0 +1,450 @@ +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use super::*; +use crate::storage::{BackendLabel, BoxedEventStore, QueryPlaneStore}; +use temper_runtime::persistence::{ + EntityKeyLookup, JournalBoundary, PersistenceAppend, PersistenceAppendResult, PersistenceError, +}; + +mod catalog_deleted; +mod complete_coverage; + +pub(super) fn complete_field_update( + persistence_id: &str, + entity_id: &str, + workspace: &str, + path: &str, + token: &str, +) -> PersistenceEnvelope { + let mut event = field_update_event(persistence_id, path, token); + event.payload["fields"] = serde_json::json!({ + "Id": entity_id, + "WorkspaceId": workspace, + "Path": path, + }); + event +} + +pub(super) async fn read_path( + state: &ServerState, + tenant: &TenantId, + workspace: &str, + path: &str, +) -> Result { + let options = QueryOptions { + filter: Some(ws_path_filter(workspace, path)), + ..QueryOptions::default() + }; + let security_ctx = SecurityContext::system(); + read_entity_set_page(QueryPlaneReadRequest { + state, + tenant, + security_ctx: &security_ctx, + entity_type: "Order", + entity_set_name: "Orders", + query_options: &options, + budget: QueryPlaneReadBudget { + default_page_size: 10, + max_entities: 10, + }, + }) + .await +} + +fn expect_read( + result: Result, + message: &str, +) -> QueryPlaneReadResult { + match result { + Ok(result) => result, + Err(error) => panic!("{message}: {}", query_read_error_name(&error)), + } +} + +fn query_read_error_name(error: &QueryPlaneReadError) -> &'static str { + match error { + QueryPlaneReadError::AuthorizationDenied(_) => "AuthorizationDenied", + QueryPlaneReadError::QueryTooLarge { .. } => "QueryTooLarge", + QueryPlaneReadError::KeyOwnershipUnstable => "KeyOwnershipUnstable", + QueryPlaneReadError::ProjectionUnstable => "ProjectionUnstable", + QueryPlaneReadError::InvalidContinuation => "InvalidContinuation", + } +} + +#[tokio::test] +async fn incomplete_scan_prefers_equal_sequence_journal_over_snapshot_source() { + let (_guard, _clock, _ids) = install_deterministic_context(252); + let tenant = TenantId::default(); + let workspace = "ws-source-replacement"; + let snapshot_path = "/snapshot-generation"; + let journal_path = "/journal-generation"; + let entity_id = "ord-equal-sequence-source"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let (state, store) = build_order_state_with_sim("incomplete-source-replacement"); + + EventStore::save_snapshot( + &store, + &persistence_id, + 1, + &snapshot(entity_id, workspace, snapshot_path), + ) + .await + .expect("seed snapshot-only generation"); + EventStore::append_with_index_rows( + &store, + &persistence_id, + 0, + &[complete_field_update( + &persistence_id, + entity_id, + workspace, + journal_path, + "replace-snapshot-source", + )], + &[key_row(workspace, journal_path)], + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(ORDER_KEY_SET_SIGNATURE.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .expect("commit equal-sequence journal replacement"); + + let old = expect_read( + read_path(&state, &tenant, workspace, snapshot_path).await, + "source-replacement old-key read", + ); + assert!( + old.entities.is_empty(), + "a journal generation must replace equal-sequence snapshot ownership" + ); + let current = expect_read( + read_path(&state, &tenant, workspace, journal_path).await, + "source-replacement current-key read", + ); + assert_eq!(current.entities.len(), 1); + assert_eq!(current.entities[0]["entity_id"], entity_id); + assert_eq!(current.entities[0]["fields"]["Path"], journal_path); + assert_eq!(current.entities[0]["sequence_nr"], 1); +} + +#[derive(Clone, Copy)] +enum BoundaryMutationMode { + ReturnCapturedBoundary, + ReturnCurrentBoundary, +} + +#[derive(Clone)] +struct BoundaryMutationStore { + inner: SimEventStore, + persistence_id: String, + entity_id: String, + workspace: String, + path: String, + expected_sequence: u64, + trigger_call: usize, + mode: BoundaryMutationMode, + boundary_calls: Arc, +} + +impl BoundaryMutationStore { + async fn append_mutation(&self) -> Result<(), PersistenceError> { + EventStore::append_with_index_rows( + &self.inner, + &self.persistence_id, + self.expected_sequence, + &[complete_field_update( + &self.persistence_id, + &self.entity_id, + &self.workspace, + &self.path, + "boundary-race", + )], + &[key_row(&self.workspace, &self.path)], + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(ORDER_KEY_SET_SIGNATURE.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await?; + Ok(()) + } +} + +impl EventStore for BoundaryMutationStore { + fn supports_authoritative_key_index(&self) -> bool { + true + } + + async fn append( + &self, + persistence_id: &str, + expected_sequence: u64, + events: &[PersistenceEnvelope], + ) -> Result { + EventStore::append(&self.inner, persistence_id, expected_sequence, events).await + } + + async fn append_batch( + &self, + appends: &[PersistenceAppend], + ) -> Result, PersistenceError> { + EventStore::append_batch(&self.inner, appends).await + } + + async fn read_events( + &self, + persistence_id: &str, + from_sequence: u64, + ) -> Result, PersistenceError> { + EventStore::read_events(&self.inner, persistence_id, from_sequence).await + } + + async fn read_events_page( + &self, + persistence_id: &str, + from_sequence: u64, + through_sequence: u64, + limit: usize, + ) -> Result, PersistenceError> { + EventStore::read_events_page( + &self.inner, + persistence_id, + from_sequence, + through_sequence, + limit, + ) + .await + } + + async fn journal_boundary( + &self, + persistence_id: &str, + ) -> Result { + let captured = EventStore::journal_boundary(&self.inner, persistence_id).await?; + let call = self.boundary_calls.fetch_add(1, Ordering::SeqCst) + 1; + if persistence_id == self.persistence_id && call == self.trigger_call { + self.append_mutation().await?; + return match self.mode { + BoundaryMutationMode::ReturnCapturedBoundary => Ok(captured), + BoundaryMutationMode::ReturnCurrentBoundary => { + EventStore::journal_boundary(&self.inner, persistence_id).await + } + }; + } + Ok(captured) + } + + async fn save_snapshot( + &self, + persistence_id: &str, + sequence_nr: u64, + snapshot: &[u8], + ) -> Result<(), PersistenceError> { + EventStore::save_snapshot(&self.inner, persistence_id, sequence_nr, snapshot).await + } + + async fn load_snapshot( + &self, + persistence_id: &str, + ) -> Result)>, PersistenceError> { + EventStore::load_snapshot(&self.inner, persistence_id).await + } + + async fn list_entity_ids( + &self, + tenant: &str, + ) -> Result, PersistenceError> { + EventStore::list_entity_ids(&self.inner, tenant).await + } + + async fn list_entity_ids_by_type( + &self, + tenant: &str, + entity_type: &str, + ) -> Result, PersistenceError> { + EventStore::list_entity_ids_by_type(&self.inner, tenant, entity_type).await + } + + async fn lookup_by_key( + &self, + tenant: &str, + entity_type: &str, + key_name: &str, + key_hash: &str, + ) -> Result, PersistenceError> { + EventStore::lookup_by_key(&self.inner, tenant, entity_type, key_name, key_hash).await + } + + async fn lookup_by_key_with_sequence( + &self, + tenant: &str, + entity_type: &str, + key_name: &str, + key_hash: &str, + ) -> Result, PersistenceError> { + EventStore::lookup_by_key_with_sequence( + &self.inner, + tenant, + entity_type, + key_name, + key_hash, + ) + .await + } + + async fn key_index_backfilled_types( + &self, + tenant: &str, + ) -> Result, PersistenceError> { + EventStore::key_index_backfilled_types(&self.inner, tenant).await + } + + async fn key_index_reconciliation_revision( + &self, + tenant: &str, + entity_type: &str, + ) -> Result { + EventStore::key_index_reconciliation_revision(&self.inner, tenant, entity_type).await + } +} + +fn install_boundary_store( + state: &mut ServerState, + store: BoundaryMutationStore, + query_plane: Option>, +) { + state.set_storage_stack(StorageStack::new( + BackendLabel::Sim, + BoxedEventStore::new(store), + None, + None, + None, + None, + query_plane, + None, + None, + None, + )); +} + +#[tokio::test] +async fn journal_zero_actor_source_is_closed_by_a_second_boundary_read() { + let (_guard, _clock, _ids) = install_deterministic_context(253); + let tenant = TenantId::default(); + let workspace = "ws-zero-source-race"; + let snapshot_path = "/snapshot-only"; + let journal_path = "/first-journal"; + let entity_id = "ord-zero-source-race"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let (mut state, inner) = build_order_state_with_sim("incomplete-zero-source-race"); + + EventStore::save_snapshot( + &inner, + &persistence_id, + 1, + &snapshot(entity_id, workspace, snapshot_path), + ) + .await + .expect("seed snapshot-only actor source"); + state + .get_or_create_tenant_entity(&tenant, "Order", entity_id, serde_json::json!({})) + .await + .expect("hydrate resident actor from snapshot-only state"); + let boundary_calls = Arc::new(AtomicUsize::new(0)); + install_boundary_store( + &mut state, + BoundaryMutationStore { + inner, + persistence_id, + entity_id: entity_id.to_string(), + workspace: workspace.to_string(), + path: journal_path.to_string(), + expected_sequence: 0, + trigger_call: 1, + mode: BoundaryMutationMode::ReturnCapturedBoundary, + boundary_calls: boundary_calls.clone(), + }, + None, + ); + + let current = expect_read( + read_path(&state, &tenant, workspace, journal_path).await, + "journal-zero source transition must stabilize", + ); + assert_eq!(current.entities.len(), 1); + assert_eq!(current.entities[0]["entity_id"], entity_id); + assert_eq!(current.entities[0]["fields"]["Path"], journal_path); + assert!( + boundary_calls.load(Ordering::SeqCst) >= 2, + "the compatibility actor source must be closed by a journal re-read" + ); +} + +#[tokio::test] +async fn incomplete_scan_retries_when_journal_advances_after_replay() { + let (_guard, _clock, _ids) = install_deterministic_context(254); + let tenant = TenantId::default(); + let workspace = "ws-post-replay-race"; + let initial_path = "/initial"; + let raced_path = "/advanced"; + let entity_id = "ord-post-replay-race"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let inner = SimEventStore::no_faults(254); + EventStore::append_with_index_rows( + &inner, + &persistence_id, + 0, + &[complete_field_update( + &persistence_id, + entity_id, + workspace, + initial_path, + "initial-journal", + )], + &[key_row(workspace, initial_path)], + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(ORDER_KEY_SET_SIGNATURE.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .expect("seed initial journal generation"); + let boundary_calls = Arc::new(AtomicUsize::new(0)); + let racing = BoundaryMutationStore { + inner, + persistence_id, + entity_id: entity_id.to_string(), + workspace: workspace.to_string(), + path: raced_path.to_string(), + expected_sequence: 1, + trigger_call: 3, + mode: BoundaryMutationMode::ReturnCurrentBoundary, + boundary_calls: boundary_calls.clone(), + }; + let mut state = build_order_state("incomplete-post-replay-race"); + install_boundary_store(&mut state, racing, None); + + let current = expect_read( + read_path(&state, &tenant, workspace, raced_path).await, + "journal advance must retry to a stable generation", + ); + assert_eq!(current.entities.len(), 1); + assert_eq!(current.entities[0]["fields"]["Path"], raced_path); + assert_eq!(current.entities[0]["sequence_nr"], 2); + assert_eq!( + boundary_calls.load(Ordering::SeqCst), + 6, + "each replay generation must be closed both before and after projection repair" + ); +} diff --git a/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/ownership_race/incomplete_coverage/source_transitions/catalog_deleted.rs b/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/ownership_race/incomplete_coverage/source_transitions/catalog_deleted.rs new file mode 100644 index 000000000..2c6a81e92 --- /dev/null +++ b/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/ownership_race/incomplete_coverage/source_transitions/catalog_deleted.rs @@ -0,0 +1,69 @@ +use super::*; + +#[tokio::test] +async fn deleted_catalog_absence_is_closed_against_first_journal_generation() { + let (_guard, _clock, _ids) = install_deterministic_context(258); + let tenant = TenantId::default(); + let workspace = "ws-deleted-catalog-source"; + let stale_path = "/deleted-catalog"; + let journal_path = "/live-journal"; + let entity_id = "ord-deleted-catalog-source"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let inner = SimEventStore::no_faults(258); + let stale_snapshot = serde_json::json!({ + "entity_type": "Order", + "entity_id": entity_id, + "status": "Deleted", + "item_count": 0, + "fields": { + "Id": entity_id, + "Status": "Deleted", + "WorkspaceId": workspace, + "Path": stale_path, + }, + }); + EventStore::save_snapshot( + &inner, + &persistence_id, + 1, + &serde_json::to_vec(&stale_snapshot).expect("serialize deleted snapshot"), + ) + .await + .expect("seed deleted snapshot-only source"); + let query_plane = Arc::new(SimQueryPlane::default()); + QueryPlaneStore::upsert_projection( + query_plane.as_ref(), + tenant.as_str(), + "Order", + entity_id, + "Deleted", + &stale_snapshot["fields"], + &stale_snapshot, + 1, + ) + .await + .expect("seed deleted catalog compatibility row"); + let boundary_calls = Arc::new(AtomicUsize::new(0)); + let racing = BoundaryMutationStore { + inner, + persistence_id, + entity_id: entity_id.to_string(), + workspace: workspace.to_string(), + path: journal_path.to_string(), + expected_sequence: 0, + trigger_call: 1, + mode: BoundaryMutationMode::ReturnCapturedBoundary, + boundary_calls: boundary_calls.clone(), + }; + let mut state = build_order_state("deleted-catalog-source-transition"); + install_boundary_store(&mut state, racing, Some(query_plane)); + + let current = expect_read( + read_path(&state, &tenant, workspace, journal_path).await, + "deleted catalog source transition must replay the first journal generation", + ); + assert_eq!(current.entities.len(), 1); + assert_eq!(current.entities[0]["entity_id"], entity_id); + assert_eq!(current.entities[0]["fields"]["Path"], journal_path); + assert!(boundary_calls.load(Ordering::SeqCst) >= 2); +} diff --git a/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/ownership_race/incomplete_coverage/source_transitions/complete_coverage.rs b/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/ownership_race/incomplete_coverage/source_transitions/complete_coverage.rs new file mode 100644 index 000000000..e6ff03161 --- /dev/null +++ b/crates/temper-server/src/odata/query_plane_read/tests/keyed_existence/ownership_race/incomplete_coverage/source_transitions/complete_coverage.rs @@ -0,0 +1,341 @@ +use std::sync::Arc; + +use super::*; + +fn build_complete_state_with_catalog( + system_name: &str, + seed: u64, +) -> (ServerState, SimEventStore, Arc) { + let store = SimEventStore::no_faults(seed); + let query_plane = Arc::new(SimQueryPlane::default()); + let mut state = build_order_state(system_name); + let mut storage = StorageStack::from_sim(store.clone(), None); + storage.query_plane = Some(query_plane.clone()); + state.set_storage_stack(storage); + (state, store, query_plane) +} + +async fn seed_live_catalog( + query_plane: &SimQueryPlane, + tenant: &TenantId, + entity_id: &str, + workspace: &str, + path: &str, + sequence_nr: u64, +) { + let fields = serde_json::json!({ + "Id": entity_id, + "WorkspaceId": workspace, + "Path": path, + }); + QueryPlaneStore::upsert_projection( + query_plane, + tenant.as_str(), + "Order", + entity_id, + "Draft", + &fields, + &serde_json::json!({ + "entity_type": "Order", + "entity_id": entity_id, + "status": "Draft", + "fields": fields, + "sequence_nr": sequence_nr, + }), + sequence_nr, + ) + .await + .expect("seed stale live catalog generation"); +} + +fn deleted_snapshot(entity_id: &str, workspace: &str, path: &str) -> Vec { + serde_json::to_vec(&serde_json::json!({ + "entity_type": "Order", + "entity_id": entity_id, + "status": "Deleted", + "item_count": 0, + "fields": { + "Id": entity_id, + "Status": "Deleted", + "WorkspaceId": workspace, + "Path": path, + }, + })) + .expect("serialize deleted snapshot") +} + +#[tokio::test] +async fn complete_key_lookup_prefers_equal_sequence_journal_over_snapshot_source() { + let (_guard, _clock, _ids) = install_deterministic_context(257); + let tenant = TenantId::default(); + let workspace = "ws-complete-source-replacement"; + let snapshot_path = "/complete-snapshot-generation"; + let journal_path = "/complete-journal-generation"; + let entity_id = "ord-complete-equal-sequence-source"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let (state, store) = build_order_state_with_sim("complete-source-replacement"); + + EventStore::save_snapshot( + &store, + &persistence_id, + 1, + &snapshot(entity_id, workspace, snapshot_path), + ) + .await + .expect("seed snapshot-only complete generation"); + EventStore::append_with_index_rows( + &store, + &persistence_id, + 0, + &[complete_field_update( + &persistence_id, + entity_id, + workspace, + journal_path, + "replace-complete-snapshot-source", + )], + &[key_row(workspace, journal_path)], + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(ORDER_KEY_SET_SIGNATURE.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .expect("commit equal-sequence complete journal replacement"); + EventStore::mark_key_index_backfilled( + &store, + tenant.as_str(), + "Order", + ORDER_KEY_SET_SIGNATURE, + ) + .await + .expect("mark complete key coverage"); + + let current = expect_read( + read_path(&state, &tenant, workspace, journal_path).await, + "complete source-replacement current-key read", + ); + assert_eq!(current.entities.len(), 1); + assert_eq!(current.entities[0]["entity_id"], entity_id); + assert_eq!(current.entities[0]["fields"]["Path"], journal_path); + assert_eq!(current.entities[0]["sequence_nr"], 1); +} + +#[tokio::test] +async fn complete_key_lookup_does_not_revive_a_mismatched_snapshot_from_catalog() { + let (_guard, _clock, _ids) = install_deterministic_context(274); + let tenant = TenantId::default(); + let workspace = "ws-complete-snapshot-mismatch"; + let stale_path = "/stale-catalog-owner"; + let current_path = "/current-snapshot-owner"; + let entity_id = "ord-complete-snapshot-mismatch"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let (state, store, query_plane) = + build_complete_state_with_catalog("complete-snapshot-mismatch", 274); + + EventStore::save_snapshot( + &store, + &persistence_id, + 3, + &snapshot(entity_id, workspace, stale_path), + ) + .await + .expect("seed stale snapshot ownership generation"); + state.populate_key_index_from_snapshots(&tenant).await; + EventStore::save_snapshot( + &store, + &persistence_id, + 3, + &snapshot(entity_id, workspace, current_path), + ) + .await + .expect("replace ownership with a same-sequence imported snapshot generation"); + seed_live_catalog(&query_plane, &tenant, entity_id, workspace, stale_path, 3).await; + EventStore::mark_key_index_backfilled( + &store, + tenant.as_str(), + "Order", + ORDER_KEY_SET_SIGNATURE, + ) + .await + .expect("restore migrated complete-coverage marker"); + + match read_path(&state, &tenant, workspace, stale_path).await { + Err(QueryPlaneReadError::KeyOwnershipUnstable) => {} + Ok(result) => panic!( + "an already-present snapshot must outrank a matching stale catalog/key generation and fail the inconsistent owner closed; returned entities: {:?}", + result.entities + ), + Err(error) => panic!( + "an already-present snapshot must fail closed as KeyOwnershipUnstable, got {}", + query_read_error_name(&error) + ), + } +} + +#[tokio::test] +async fn complete_key_lookup_does_not_revive_a_deleted_snapshot_from_catalog() { + let (_guard, _clock, _ids) = install_deterministic_context(275); + let tenant = TenantId::default(); + let workspace = "ws-complete-deleted-snapshot"; + let path = "/deleted-snapshot-owner"; + let entity_id = "ord-complete-deleted-snapshot"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let (state, store, query_plane) = + build_complete_state_with_catalog("complete-deleted-snapshot", 275); + + EventStore::save_snapshot( + &store, + &persistence_id, + 3, + &snapshot(entity_id, workspace, path), + ) + .await + .expect("seed stale live snapshot ownership"); + state.populate_key_index_from_snapshots(&tenant).await; + EventStore::save_snapshot( + &store, + &persistence_id, + 3, + &deleted_snapshot(entity_id, workspace, path), + ) + .await + .expect("replace durable source with a terminal snapshot"); + seed_live_catalog(&query_plane, &tenant, entity_id, workspace, path, 3).await; + EventStore::mark_key_index_backfilled( + &store, + tenant.as_str(), + "Order", + ORDER_KEY_SET_SIGNATURE, + ) + .await + .expect("restore migrated complete-coverage marker"); + + assert!( + matches!( + read_path(&state, &tenant, workspace, path).await, + Err(QueryPlaneReadError::KeyOwnershipUnstable) + ), + "a terminal snapshot must never be revived by a stale live catalog row and must fail the inconsistent owner closed" + ); +} + +#[tokio::test] +async fn catalog_only_owner_must_match_the_requested_declared_key() { + let (_guard, _clock, _ids) = install_deterministic_context(276); + let tenant = TenantId::default(); + let workspace = "ws-catalog-only-key-mismatch"; + let indexed_path = "/indexed-owner"; + let catalog_path = "/different-catalog-body"; + let entity_id = "ord-catalog-only-key-mismatch"; + let (state, store, query_plane) = + build_complete_state_with_catalog("catalog-only-key-mismatch", 276); + + let contract_revision = EventStore::begin_key_index_backfill( + &store, + tenant.as_str(), + "Order", + ORDER_KEY_SET_SIGNATURE, + ) + .await + .expect("begin catalog-only key contract"); + EventStore::backfill_entity_keys( + &store, + tenant.as_str(), + "Order", + entity_id, + 0, + KeyIndexBackfillFence { + key_set_signature: ORDER_KEY_SET_SIGNATURE, + contract_revision, + expected_journal_sequence: 0, + expected_entity_live: false, + expected_snapshot: None, + }, + &[key_row(workspace, indexed_path)], + ) + .await + .expect("seed catalog-only ownership row"); + EventStore::mark_key_index_backfilled_if_revision( + &store, + tenant.as_str(), + "Order", + ORDER_KEY_SET_SIGNATURE, + contract_revision, + ) + .await + .expect("complete catalog-only key coverage"); + seed_live_catalog(&query_plane, &tenant, entity_id, workspace, catalog_path, 0).await; + + assert!( + matches!( + read_path(&state, &tenant, workspace, indexed_path).await, + Err(QueryPlaneReadError::KeyOwnershipUnstable) + ), + "a complete key row must not certify a catalog-only body whose declared-key fields hash differently" + ); +} + +#[tokio::test] +async fn complete_key_read_ignores_unrelated_over_budget_projection_debt() { + let (_guard, _clock, _ids) = install_deterministic_context(277); + let tenant = TenantId::default(); + let workspace = "ws-keyed-through-dirty-debt"; + let path = "/authoritative-owner"; + let entity_id = "ord-keyed-through-dirty-debt"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let (state, store, query_plane) = + build_complete_state_with_catalog("keyed-through-dirty-debt", 277); + + EventStore::append_with_index_rows( + &store, + &persistence_id, + 0, + &[complete_field_update( + &persistence_id, + entity_id, + workspace, + path, + "seed-keyed-through-dirty-debt", + )], + &[key_row(workspace, path)], + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(ORDER_KEY_SET_SIGNATURE.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .expect("seed complete authoritative owner"); + EventStore::mark_key_index_backfilled( + &store, + tenant.as_str(), + "Order", + ORDER_KEY_SET_SIGNATURE, + ) + .await + .expect("publish complete key coverage"); + + for index in 0..=100 { + query_plane.mark_dirty("Order", &format!("unrelated-dirty-{index:03}")); + } + assert_eq!(query_plane.dirty_count(), 101); + + let result = expect_read( + read_path(&state, &tenant, workspace, path).await, + "complete keyed read with unrelated projection debt", + ); + assert_eq!(result.entities.len(), 1); + assert_eq!(result.entities[0]["entity_id"], entity_id); + assert_eq!(result.entities[0]["fields"]["Path"], path); + assert_eq!( + query_plane.dirty_count(), + 101, + "an authoritative key read must not drain unrelated catalog/EAV repair debt" + ); +} diff --git a/crates/temper-server/src/odata/query_plane_read/tests/paging.rs b/crates/temper-server/src/odata/query_plane_read/tests/paging.rs index 37e7fc291..6eae62614 100644 --- a/crates/temper-server/src/odata/query_plane_read/tests/paging.rs +++ b/crates/temper-server/src/odata/query_plane_read/tests/paging.rs @@ -130,24 +130,15 @@ async fn turso_state(system_name: &str) -> (ServerState, TursoEventStore, String /// project its test fields into the catalog. async fn seed_order( state: &ServerState, - store: &TursoEventStore, + _store: &TursoEventStore, tenant: &TenantId, id: &str, fields: serde_json::Value, ) { - let agent_ctx = AgentContext::for_service("paging-test"); state - .dispatch_tenant_action( - tenant, - "Order", - id, - "Create", - serde_json::json!({}), - &agent_ctx, - ) + .get_or_create_tenant_entity(tenant, "Order", id, fields) .await .expect("create order"); - upsert_order_projection(store, tenant, id, fields, 1).await; } /// Unfiltered list: the default entity_id-ascending order paginates cleanly and diff --git a/crates/temper-server/src/odata/query_plane_read/tests/proof.rs b/crates/temper-server/src/odata/query_plane_read/tests/proof.rs index 09d0d7194..7f2ab3a9f 100644 --- a/crates/temper-server/src/odata/query_plane_read/tests/proof.rs +++ b/crates/temper-server/src/odata/query_plane_read/tests/proof.rs @@ -5,8 +5,9 @@ use super::*; /// `entity_key_index` — a bounded candidate (no full-type scan → the budget that /// raises the 413 can never trip). Gated on DATABASE_URL; unique tenant; cleans up. /// -/// Proves `keyed_candidate_ids` composes the verified pieces against a live DB: -/// real `append_with_keys` co-commit → real `lookup_by_key` → bounded `[id]`. +/// Proves `resolve_keyed_candidates` composes the verified pieces against a live +/// DB: real `append_with_keys` co-commit → v3 watermark → real `lookup_by_key` +/// → bounded `[id]`. #[test] fn keyed_filter_resolves_to_bounded_candidate_on_postgres() { use temper_runtime::persistence::{ @@ -83,6 +84,9 @@ fn keyed_filter_resolves_to_bounded_candidate_on_postgres() { ) .await .expect("co-commit key row"); + state + .mark_key_index_backfilled(&tenant, "Order", ORDER_KEY_SET_SIGNATURE) + .await; // AMPLIFICATION BOUND (ADR-0148 not regressed): the synchronous co-commit // wrote exactly K=1 key row and ZERO broad-index rows. The S-wide @@ -146,11 +150,14 @@ fn keyed_filter_resolves_to_bounded_candidate_on_postgres() { max_entities: 10, }, }; - assert_eq!( - keyed_candidate_ids(&request).await, - Some(vec![target_id.to_string()]), - "a $filter matching the declared key must resolve to the bounded single id via entity_key_index" - ); + match resolve_keyed_candidates(&request).await { + KeyedCandidateResolution::Authoritative(proof) => { + assert_eq!(proof.owner(), Some(target_id)); + } + other => { + panic!("a declared-key filter must resolve through entity_key_index, got {other:?}") + } + } // CONTROL: a non-key filter does not resolve -> None (falls back to scan). let non_key = QueryOptions { @@ -168,14 +175,14 @@ fn keyed_filter_resolves_to_bounded_candidate_on_postgres() { ..request }; assert_eq!( - keyed_candidate_ids(&control).await, - None, + resolve_keyed_candidates(&control).await, + KeyedCandidateResolution::NotApplicable, "a non-key filter must decline the keyed fast path" ); // ABSENT key: a declared-key filter for a key NO entity holds resolves to a - // keyed MISS. Pre-backfill that returns None (fall back to scan), NOT an - // authoritative empty — a missing key row may be a not-yet-backfilled entity. + // keyed MISS. Once the v3 watermark covers the current declaration, that + // miss is an authoritative empty candidate set. let absent = QueryOptions { filter: Some(FilterExpr::BinaryOp { left: Box::new(FilterExpr::BinaryOp { @@ -198,11 +205,14 @@ fn keyed_filter_resolves_to_bounded_candidate_on_postgres() { query_options: &absent, ..request }; - assert_eq!( - keyed_candidate_ids(&absent_req).await, - None, - "a keyed miss must decline to scan fallback pre-backfill (not authoritative-empty)" - ); + match resolve_keyed_candidates(&absent_req).await { + KeyedCandidateResolution::Authoritative(proof) => { + assert_eq!(proof.owner(), None); + } + other => panic!( + "a keyed miss must be authoritative after the current v3 watermark, got {other:?}" + ), + } // Clean up this run's rows. let _ = sqlx::query("DELETE FROM entity_key_index WHERE tenant = $1") @@ -216,10 +226,11 @@ fn keyed_filter_resolves_to_bounded_candidate_on_postgres() { }); } -/// ADR-0153 boundary: the keyed fast path engages ONLY for a pure-equality filter -/// that is exactly a declared `[[key]]`. Every other shape must decline (return -/// None) so the read falls back to the existing scan/pushdown path — no behavior -/// change, no false hit. No DB needed: all these decline before any store call. +/// ADR-0153/0171 boundary: key ownership applies only to a pure-equality filter +/// that is exactly a declared `[[key]]`. Non-key filter shapes decline. Query +/// modifiers such as `$orderby` and `$count` do not change ownership: they retain +/// the exact-key authority boundary and honor their semantics over the bounded +/// zero-or-one result. No DB is needed for these shape checks. #[tokio::test] async fn keyed_fast_path_declines_non_key_shapes() { let mut state = build_order_state("query-plane-keyed-decline"); @@ -299,18 +310,6 @@ async fn keyed_fast_path_declines_non_key_shapes() { ..QueryOptions::default() }, ), - // $orderby present (a point read has no ordering to honor) - ( - "with_orderby", - QueryOptions { - filter: Some(and(eq("WorkspaceId", "ws"), eq("Path", "/a"))), - orderby: Some(vec![OrderByClause { - property: "Path".to_string(), - direction: OrderDirection::Asc, - }]), - ..QueryOptions::default() - }, - ), ]; for (name, query_options) in &cases { @@ -324,11 +323,43 @@ async fn keyed_fast_path_declines_non_key_shapes() { budget, }; assert_eq!( - keyed_candidate_ids(&request).await, - None, + resolve_keyed_candidates(&request).await, + KeyedCandidateResolution::NotApplicable, "keyed fast path must decline shape '{name}' (falls back to scan/pushdown)" ); } + + let exact_key_with_modifiers = [ + QueryOptions { + filter: Some(and(eq("WorkspaceId", "ws"), eq("Path", "/a"))), + orderby: Some(vec![OrderByClause { + property: "Path".to_string(), + direction: OrderDirection::Asc, + }]), + ..QueryOptions::default() + }, + QueryOptions { + filter: Some(and(eq("WorkspaceId", "ws"), eq("Path", "/a"))), + count: Some(true), + ..QueryOptions::default() + }, + ]; + for query_options in &exact_key_with_modifiers { + let request = QueryPlaneReadRequest { + state: &state, + tenant: &tenant, + security_ctx: &security_ctx, + entity_type: "Order", + entity_set_name: "Orders", + query_options, + budget, + }; + assert_eq!( + resolve_keyed_candidates(&request).await, + KeyedCandidateResolution::NeedsAuthoritativeScan, + "exact-key modifiers must preserve the ownership authority boundary" + ); + } } async fn upsert_order_projection_with_status( @@ -423,6 +454,7 @@ fn native_plan_pushes_file_lookup_equalities_with_status_ne_recheck() { #[tokio::test] async fn ordinary_null_filter_uses_lossless_candidate_scan() { + let _turso_test_guard = TURSO_QUERY_PROOF_LOCK.lock().await; let db_path = std::env::temp_dir().join(format!("temper-query-plane-null-proof-{}.db", sim_uuid())); let _ = std::fs::remove_file(&db_path); @@ -522,6 +554,7 @@ async fn ordinary_null_filter_uses_lossless_candidate_scan() { #[tokio::test] async fn nullable_scalar_order_uses_read_source_full_proof() { + let _turso_test_guard = TURSO_QUERY_PROOF_LOCK.lock().await; let db_path = std::env::temp_dir().join(format!("temper-query-plane-null-order-{}.db", sim_uuid())); let _ = std::fs::remove_file(&db_path); @@ -596,6 +629,7 @@ async fn nullable_scalar_order_uses_read_source_full_proof() { #[tokio::test] async fn context_prep_shaped_filter_with_huge_top_uses_bounded_native_page() { + let _turso_test_guard = TURSO_QUERY_PROOF_LOCK.lock().await; let db_path = std::env::temp_dir().join(format!( "temper-query-plane-session-filter-{}.db", sim_uuid() @@ -609,19 +643,23 @@ async fn context_prep_shaped_filter_with_huge_top_uses_bounded_native_page() { state.set_storage_stack(StorageStack::from_turso(store.clone())); let tenant = TenantId::default(); - for index in 0usize..1200 { - upsert_order_projection( - &store, - &tenant, - &format!("entry-{index:04}"), - serde_json::json!({ - "SessionId": "session-hot", - "ParentEntryId": format!("entry-{:04}", index.saturating_sub(1)), - }), - index as u64 + 1, - ) - .await; - } + upsert_order_projections( + &store, + &tenant, + (0usize..1200) + .map(|index| { + ( + format!("entry-{index:04}"), + serde_json::json!({ + "SessionId": "session-hot", + "ParentEntryId": format!("entry-{:04}", index.saturating_sub(1)), + }), + index as u64 + 1, + ) + }) + .collect(), + ) + .await; upsert_order_projection( &store, &tenant, @@ -679,6 +717,7 @@ async fn context_prep_shaped_filter_with_huge_top_uses_bounded_native_page() { #[tokio::test] async fn file_point_lookup_with_status_ne_uses_lossless_equality_candidates() { + let _turso_test_guard = TURSO_QUERY_PROOF_LOCK.lock().await; let db_path = std::env::temp_dir().join(format!( "temper-query-plane-file-status-ne-{}.db", sim_uuid() @@ -820,6 +859,12 @@ async fn file_point_lookup_with_status_ne_uses_lossless_equality_candidates() { Err(QueryPlaneReadError::InvalidContinuation) => { panic!("no $skiptoken was supplied") } + Err(QueryPlaneReadError::KeyOwnershipUnstable) => { + panic!("file point lookup ownership should remain stable") + } + Err(QueryPlaneReadError::ProjectionUnstable) => { + panic!("file point lookup projection should remain stable") + } }; assert_eq!(result.entities.len(), 1); @@ -844,6 +889,7 @@ async fn file_point_lookup_with_status_ne_uses_lossless_equality_candidates() { #[tokio::test] async fn session_entry_chain_parent_lookup_uses_bounded_native_page() { + let _turso_test_guard = TURSO_QUERY_PROOF_LOCK.lock().await; let db_dir = tempfile::tempdir().expect("create isolated query-plane db dir"); let db_path = db_dir.path().join("session-parent.db"); let db_url = format!("file:{}", db_path.display()); @@ -854,19 +900,23 @@ async fn session_entry_chain_parent_lookup_uses_bounded_native_page() { state.set_storage_stack(StorageStack::from_turso(store.clone())); let tenant = TenantId::default(); - for index in 0usize..1200 { - upsert_order_projection( - &store, - &tenant, - &format!("entry-{index:04}"), - serde_json::json!({ - "SessionId": "session-hot", - "ParentEntryId": format!("entry-{:04}", index.saturating_sub(1)), - }), - index as u64 + 1, - ) - .await; - } + upsert_order_projections( + &store, + &tenant, + (0usize..1200) + .map(|index| { + ( + format!("entry-{index:04}"), + serde_json::json!({ + "SessionId": "session-hot", + "ParentEntryId": format!("entry-{:04}", index.saturating_sub(1)), + }), + index as u64 + 1, + ) + }) + .collect(), + ) + .await; let security_ctx = SecurityContext::system(); let query_options = QueryOptions { @@ -921,6 +971,7 @@ async fn session_entry_chain_parent_lookup_uses_bounded_native_page() { #[tokio::test] async fn session_entry_leaf_id_lookup_uses_bounded_native_page() { + let _turso_test_guard = TURSO_QUERY_PROOF_LOCK.lock().await; let db_dir = tempfile::tempdir().expect("create isolated query-plane db dir"); let db_path = db_dir.path().join("session-leaf.db"); let db_url = format!("file:{}", db_path.display()); @@ -931,19 +982,23 @@ async fn session_entry_leaf_id_lookup_uses_bounded_native_page() { state.set_storage_stack(StorageStack::from_turso(store.clone())); let tenant = TenantId::default(); - for index in 0usize..1200 { - upsert_order_projection( - &store, - &tenant, - &format!("entry-{index:04}"), - serde_json::json!({ - "SessionId": "session-hot", - "ParentEntryId": format!("entry-{:04}", index.saturating_sub(1)), - }), - index as u64 + 1, - ) - .await; - } + upsert_order_projections( + &store, + &tenant, + (0usize..1200) + .map(|index| { + ( + format!("entry-{index:04}"), + serde_json::json!({ + "SessionId": "session-hot", + "ParentEntryId": format!("entry-{:04}", index.saturating_sub(1)), + }), + index as u64 + 1, + ) + }) + .collect(), + ) + .await; let security_ctx = SecurityContext::system(); let query_options = QueryOptions { @@ -994,6 +1049,7 @@ async fn session_entry_leaf_id_lookup_uses_bounded_native_page() { #[tokio::test] async fn unsafe_order_uses_read_source_full_proof() { + let _turso_test_guard = TURSO_QUERY_PROOF_LOCK.lock().await; let db_path = std::env::temp_dir().join(format!("temper-query-plane-order-proof-{}.db", sim_uuid())); let _ = std::fs::remove_file(&db_path); @@ -1077,30 +1133,26 @@ async fn build_large_projected_type( ) -> ServerState { let mut state = build_order_state(system_name); state.set_storage_stack(StorageStack::from_turso(store.clone())); - let agent_ctx = AgentContext::for_service(system_name); for index in 0..count { let entity_id = format!("entry-{index:04}"); state - .dispatch_tenant_action( + .get_or_create_tenant_entity( tenant, "Order", &entity_id, - "Create", - serde_json::json!({}), - &agent_ctx, + serde_json::json!({ "SessionId": "session-hot" }), ) .await .expect("create order"); - // Deterministic projected fields (high sequence wins over the async queue). - upsert_order_projection( - store, - tenant, - &entity_id, - serde_json::json!({ "SessionId": "session-hot" }), - 1_000 + index as u64, - ) - .await; } + crate::state::source_fenced_projection::repair_dirty_projections_before_read( + &state, + tenant, + "Order", + count.max(1), + ) + .await + .expect("establish a fully repaired projected type"); state } @@ -1123,6 +1175,7 @@ fn session_filter_options(session: &str) -> QueryOptions { /// Before the fix this was an unconditional budget rejection. #[tokio::test] async fn empty_equality_list_on_large_type_is_authoritative_absent_without_scan() { + let _turso_test_guard = TURSO_QUERY_PROOF_LOCK.lock().await; let db_path = std::env::temp_dir().join(format!("temper-arn68-empty-list-{}.db", sim_uuid())); let _ = std::fs::remove_file(&db_path); let db_url = format!("file:{}", db_path.display()); @@ -1164,6 +1217,7 @@ async fn empty_equality_list_on_large_type_is_authoritative_absent_without_scan( /// stays a bounded empty answer. Before the fix both were budget rejections. #[tokio::test] async fn equality_list_on_large_type_repairs_unprojected_match_boundedly() { + let _turso_test_guard = TURSO_QUERY_PROOF_LOCK.lock().await; let db_path = std::env::temp_dir().join(format!("temper-arn68-gap-list-{}.db", sim_uuid())); let _ = std::fs::remove_file(&db_path); let db_url = format!("file:{}", db_path.display()); @@ -1176,8 +1230,20 @@ async fn equality_list_on_large_type_repairs_unprojected_match_boundedly() { // The lagging entity: journal-durable via DIRECT append — its projection was never // enqueued (the crash-lost shape, deterministic: no async worker to race). The // snapshot carries the queried SessionId for materialization. - use temper_runtime::persistence::{EventMetadata, EventStore as _, PersistenceEnvelope}; + use temper_runtime::persistence::{ + EventMetadata, EventStore as _, PersistenceEnvelope, ProjectionSourceFence, + SnapshotBackfillFence, + }; use temper_runtime::scheduler::{sim_now, sim_uuid as runtime_sim_uuid}; + let timestamp = sim_now(); + let lagging_event = crate::entity_actor::EntityEvent { + action: "Create".to_string(), + from_status: "Draft".to_string(), + to_status: "Draft".to_string(), + timestamp, + params: serde_json::json!({ "SessionId": "session-brand-new" }), + idempotency_key: None, + }; store .append( &format!("{tenant}:Order:entry-lagging"), @@ -1185,15 +1251,12 @@ async fn equality_list_on_large_type_repairs_unprojected_match_boundedly() { &[PersistenceEnvelope { sequence_nr: 1, event_type: "Create".to_string(), - payload: serde_json::json!({ - "action": "Create", - "params": { "SessionId": "session-brand-new" }, - }), + payload: serde_json::to_value(lagging_event).expect("serialize lagging event"), metadata: EventMetadata { event_id: runtime_sim_uuid(), causation_id: runtime_sim_uuid(), correlation_id: runtime_sim_uuid(), - timestamp: sim_now(), + timestamp, actor_id: "arn68-gap-list".to_string(), }, }], @@ -1207,14 +1270,29 @@ async fn equality_list_on_large_type_repairs_unprojected_match_boundedly() { "item_count": 0, "fields": { "Id": "entry-lagging", "Status": "Draft", "SessionId": "session-brand-new" }, }); + let snapshot_bytes = serde_json::to_vec(&snapshot).expect("serialize lagging snapshot"); store - .save_snapshot( - &format!("{tenant}:Order:entry-lagging"), - 1, - &serde_json::to_vec(&snapshot).unwrap(), - ) + .save_snapshot(&format!("{tenant}:Order:entry-lagging"), 1, &snapshot_bytes) .await .expect("seed snapshot"); + assert!( + store + .clear_query_projection_dirty_if_source( + tenant.as_str(), + "Order", + "entry-lagging", + ProjectionSourceFence { + expected_journal_sequence: 1, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: 1, + state: &snapshot_bytes, + }), + }, + ) + .await + .expect("clear exact lagging dirty generation"), + "the fixture must model a pre-ledger crash-lost projection" + ); let security_ctx = SecurityContext::system(); let budget = QueryPlaneReadBudget { @@ -1270,6 +1348,7 @@ async fn equality_list_on_large_type_repairs_unprojected_match_boundedly() { /// `$select` must not strip the union's `entity_id` key. #[tokio::test] async fn equality_list_rescue_honors_skip_and_select() { + let _turso_test_guard = TURSO_QUERY_PROOF_LOCK.lock().await; let db_path = std::env::temp_dir().join(format!("temper-arn68-skip-list-{}.db", sim_uuid())); let _ = std::fs::remove_file(&db_path); let db_url = format!("file:{}", db_path.display()); @@ -1279,19 +1358,25 @@ async fn equality_list_rescue_honors_skip_and_select() { let tenant = TenantId::default(); let state = build_large_projected_type(&store, &tenant, "arn68-skip-list", 15).await; - // Two PROJECTED matches for the queried session (visible only to the page)... - for (id, seq) in [("paged-c1", 5001u64), ("paged-c2", 5002)] { - upsert_order_projection( - &store, - &tenant, - id, - serde_json::json!({ "SessionId": "session-paged" }), - seq, - ) - .await; + // Two durable matches for the queried session, then source-fenced repair + // establishes their catalog rows before paging. + for id in ["paged-c1", "paged-c2"] { + state + .get_or_create_tenant_entity( + &tenant, + "Order", + id, + serde_json::json!({ "SessionId": "session-paged" }), + ) + .await + .expect("create paged match"); } - // ...and they must also be journal-durable so enumeration sees the type as-is. - // (The 15 harness entities already push the type over budget.) + crate::state::source_fenced_projection::repair_dirty_projections_before_read( + &state, &tenant, "Order", 2, + ) + .await + .expect("repair paged match projections"); + // The 15 harness entities already push the type over budget. let security_ctx = SecurityContext::system(); let budget = QueryPlaneReadBudget { diff --git a/crates/temper-server/src/odata/query_plane_read/types.rs b/crates/temper-server/src/odata/query_plane_read/types.rs index df3d28890..ddb17594d 100644 --- a/crates/temper-server/src/odata/query_plane_read/types.rs +++ b/crates/temper-server/src/odata/query_plane_read/types.rs @@ -10,6 +10,26 @@ use crate::state::ServerState; use super::super::read_support::{odata_default_page_size, odata_max_entities}; +/// Authorization boundary for one query-plane execution. +/// +/// Ordinary entity-set reads enforce collection `list` plus per-row `read`. +/// Composite-key identity resolution is an internal lookup whose returned body +/// is authorized exactly once by the enclosing entity GET. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(in crate::odata::query_plane_read) enum QueryPlaneReadAuthorization { + /// Enforce the caller's collection and row authorization throughout the query. + CallerScoped, + /// Resolve an identity internally before the enclosing GET authorizes its body. + InternalIdentityResolution, +} + +impl QueryPlaneReadAuthorization { + /// Return whether this execution must enforce the caller's query-plane checks. + pub(in crate::odata::query_plane_read) fn enforces_caller(self) -> bool { + self == Self::CallerScoped + } +} + /// Explicit budgets for one OData query-plane read. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub(in crate::odata) struct QueryPlaneReadBudget { @@ -223,6 +243,14 @@ pub(in crate::odata) enum QueryPlaneReadError { AuthorizationDenied(Box), /// The read would exceed the bounded candidate proof budget. QueryTooLarge { telemetry: QueryPlaneReadTelemetry }, + /// Ownership changed during every bounded lookup/materialization retry. + /// Returning an entity or an empty set would not have a stable ownership + /// proof, so the caller receives a retryable response instead. + KeyOwnershipUnstable, + /// A durable source changed while its query projection was being repaired, + /// or the bounded repair budget was exhausted. Native projection reads are + /// unsafe until a later retry closes that source generation. + ProjectionUnstable, /// The `$skiptoken` continuation could not be decoded for this request — /// malformed, or its key count does not match the request's ordering. InvalidContinuation, @@ -233,6 +261,8 @@ impl QueryPlaneReadError { match self { Self::AuthorizationDenied(_) => {} Self::QueryTooLarge { telemetry, .. } => telemetry.record(span), + Self::KeyOwnershipUnstable => {} + Self::ProjectionUnstable => {} Self::InvalidContinuation => {} } } @@ -246,6 +276,18 @@ impl QueryPlaneReadError { "This query requires evaluating more candidate entities than the bounded OData read budget permits. Use a narrower filter or a smaller page/count request.", ) .into_response(), + Self::KeyOwnershipUnstable => odata_error( + StatusCode::SERVICE_UNAVAILABLE, + "KeyOwnershipUnstable", + "Declared-key ownership changed while this read was materialized. Retry the request.", + ) + .into_response(), + Self::ProjectionUnstable => odata_error( + StatusCode::SERVICE_UNAVAILABLE, + "ProjectionUnstable", + "The query projection changed while this read was materialized. Retry the request.", + ) + .into_response(), Self::InvalidContinuation => odata_error( StatusCode::BAD_REQUEST, "InvalidSkipToken", diff --git a/crates/temper-server/src/odata/read.rs b/crates/temper-server/src/odata/read.rs index 3a7e36e10..d78210ccb 100644 --- a/crates/temper-server/src/odata/read.rs +++ b/crates/temper-server/src/odata/read.rs @@ -20,12 +20,14 @@ use super::common::{ check_has_stream_or_400, extract_key, extract_tenant, has_expand_options, resolve_entity_type, resolve_value_parent, tenant_csdl_xml, tenant_entity_sets, }; -use super::filter_sql; use super::query_plane_read::{ - QueryPlaneReadBudget, QueryPlaneReadRequest, read_entity_set_from_query_plane, + QueryPlaneReadBudget, QueryPlaneReadError, QueryPlaneReadRequest, + read_entity_set_for_internal_resolution, read_entity_set_from_query_plane, }; use super::read_support::{ - record_entity_set_not_found, resolve_entity_set_name, try_load_entity_body_from_catalog, + ExactEntityMaterialization, durable_source_absent_for_catalog_materialization, + materialize_exact_entity, record_entity_set_not_found, resolve_entity_set_name, + try_load_entity_body_from_catalog, }; use super::response::annotate_entity; use super::stream_fast_path::try_file_stream_fast_path; @@ -35,7 +37,6 @@ use crate::query_eval::{expand_entity, select_fields}; use crate::request_context::extract_agent_context; use crate::response::{ODataResponse, ODataStreamResponse, ODataXmlResponse, odata_error}; use crate::state::ServerState; -use crate::storage::{QueryFieldIndexOrder, QueryFieldIndexOrderDirection}; /// Recursively resolve an OData path to its parent entity's /// (entity_type, entity_id, entity_set_name). @@ -208,13 +209,10 @@ fn resource_not_found_response(set_name: &str, key: &str) -> Response { /// Load a single entity body for OData read handlers. /// -/// Tries the durable `entity_catalog` projection first when a query-plane -/// store is configured or the `TEMPER_ODATA_CATALOG_FAST_READ` flag is on. On -/// a hit, returns a body whose shape matches the actor's serialized -/// `EntityState` — blob refs already hydrated. On miss (no catalog row, -/// catalog disabled, or backend error), falls back to the actor path: validate -/// the entity exists in the in-memory index, then load via -/// `get_tenant_entity_state`. +/// Closes the read against its journal/snapshot generation before considering +/// the asynchronous `entity_catalog`. The catalog may supply a body only for an +/// ADR-0077 migration entity proven to have neither durable source. On catalog +/// miss, falls back to the actor path. /// /// The catalog-first path bypasses the in-memory index, so it survives /// cold starts and bulk-imported entities (data on disk but not yet @@ -226,11 +224,36 @@ async fn load_existing_entity_body( set_name: &str, key: &str, ) -> Result { + let catalog_compatibility = if state.event_journal().is_some() { + match materialize_exact_entity(state, tenant, entity_type, set_name, key).await { + Ok(ExactEntityMaterialization::Present(body)) => return Ok(body), + Ok(ExactEntityMaterialization::NotFound) => { + return Err(resource_not_found_response(set_name, key)); + } + Ok(ExactEntityMaterialization::CatalogCompatible) => true, + Err(_) => return Err(QueryPlaneReadError::ProjectionUnstable.into_response()), + } + } else { + false + }; + let prefer_catalog = state.query_plane_store().is_some(); if let Some(body) = try_load_entity_body_from_catalog(state, tenant, entity_type, set_name, key, prefer_catalog) .await { + if body.get("status").and_then(serde_json::Value::as_str) == Some("Deleted") { + return Err(resource_not_found_response(set_name, key)); + } + if catalog_compatibility + && !matches!( + durable_source_absent_for_catalog_materialization(state, tenant, entity_type, key,) + .await, + Ok(true) + ) + { + return Err(QueryPlaneReadError::ProjectionUnstable.into_response()); + } return Ok(body); } @@ -303,76 +326,51 @@ async fn try_resolve_composite_entity_key( tenant: &TenantId, entity_type: &str, key_pairs: &[(String, String)], -) -> Option { - // ADR-0153 fast path: if the key is a declared `[[key]]`, probe - // `entity_key_index` (O(log n), present/absent) instead of the candidate - // scan. On a miss we fall through to the scan, which still covers - // pre-backfill entities — a safe additive fast path until #324's scan is - // retired behind the backfill gate. - // Composite-key URL addressing delivers string values; carry them typed so - // `resolve_query_to_key` hashes them the same way the write side does. - let typed_pairs: Vec<(String, serde_json::Value)> = key_pairs - .iter() - .map(|(k, v)| (k.clone(), serde_json::Value::String(v.clone()))) - .collect(); - // Resolve declared keys via the registry-aware path (os-app entities live in - // the per-tenant registry, not `transition_tables`). - let keys = state.declared_keys_for(tenant, entity_type); - if let Some((key_name, key_hash)) = crate::key_index::resolve_query_to_key(&keys, &typed_pairs) - && let Some((store, _)) = state.event_journal() - && let Ok(Some(entity_id)) = store - .lookup_by_key(tenant.as_str(), entity_type, &key_name, &key_hash) - .await - { - return Some(entity_id); - } - - let query_plane = state.query_plane_store()?; - let filter = composite_key_filter(key_pairs)?; - let translated = filter_sql::try_translate_candidate_filter(&filter)?; - let order_by = [QueryFieldIndexOrder { - field_name: "entity_id".to_string(), - direction: QueryFieldIndexOrderDirection::Asc, - }]; - let page = match query_plane - .query_field_index_page( - tenant.as_str(), - entity_type, - &translated.where_clause, - translated.params, - &order_by, - 0, - 2, - false, - ) - .await - { - Ok(Some(page)) => page, - Ok(None) => return None, - Err(error) => { - tracing::warn!( - error = %error, - tenant = %tenant, - entity_type, - "composite OData key lookup failed; falling back to direct key" - ); - return None; - } +) -> Result, QueryPlaneReadError> { + // Use the entity-set exact-key contract for both ownership resolution and + // journal-backed materialization. Keeping the proven body with the resolved + // ID prevents a same-contract ownership transfer from racing a second load. + let Some(filter) = composite_key_filter(key_pairs) else { + return Ok(None); + }; + let query_options = QueryOptions { + filter: Some(filter), + top: Some(2), + ..QueryOptions::default() + }; + let entity_set_name = resolve_entity_set_name(state, tenant, entity_type); + // Composite URL resolution is an internal identity lookup, not a collection + // read. Requiring the caller's `list` permission here would make an entity GET + // stricter solely because its key is composite. The resolved body still passes + // through the caller-scoped entity `read` authorization in `build_entity_body`. + let resolution_security_ctx = SecurityContext::system(); + let result = read_entity_set_for_internal_resolution(QueryPlaneReadRequest { + state, + tenant, + security_ctx: &resolution_security_ctx, + entity_type, + entity_set_name: &entity_set_name, + query_options: &query_options, + budget: QueryPlaneReadBudget::from_config(), + }) + .await?; + let [entity] = result.entities.as_slice() else { + return Ok(None); }; + Ok(entity + .get("Id") + .or_else(|| entity.get("entity_id")) + .or_else(|| entity.get("fields").and_then(|fields| fields.get("Id"))) + .and_then(serde_json::Value::as_str) + .map(|entity_id| ResolvedEntityRequest { + entity_id: entity_id.to_string(), + authoritative_body: Some(entity.clone()), + })) +} - match page.entity_ids.as_slice() { - [entity_id] => Some(entity_id.clone()), - [] => None, - _ => { - tracing::warn!( - tenant = %tenant, - entity_type, - match_count = page.entity_ids.len(), - "composite OData key lookup was ambiguous; falling back to direct key" - ); - None - } - } +struct ResolvedEntityRequest { + entity_id: String, + authoritative_body: Option, } async fn resolve_entity_request_key( @@ -380,13 +378,21 @@ async fn resolve_entity_request_key( tenant: &TenantId, entity_type: &str, key: &KeyValue, -) -> String { +) -> Result { match key { - KeyValue::Single(_) => extract_key(key), + KeyValue::Single(_) => Ok(ResolvedEntityRequest { + entity_id: extract_key(key), + authoritative_body: None, + }), KeyValue::Composite(pairs) => { - try_resolve_composite_entity_key(state, tenant, entity_type, pairs) - .await - .unwrap_or_else(|| extract_key(key)) + Ok( + try_resolve_composite_entity_key(state, tenant, entity_type, pairs) + .await? + .unwrap_or_else(|| ResolvedEntityRequest { + entity_id: extract_key(key), + authoritative_body: None, + }), + ) } } } @@ -423,7 +429,12 @@ async fn apply_entity_query_options( Ok(body) } -struct EntityBodyOptions<'a> { +struct EntityBodyRequest<'a> { + entity_type: &'a str, + set_name: &'a str, + key: &'a str, + security_ctx: &'a SecurityContext, + authoritative_body: Option, context: String, odata_id: Option, query_options: &'a QueryOptions, @@ -435,25 +446,48 @@ struct EntityBodyOptions<'a> { async fn build_entity_body( state: &ServerState, tenant: &TenantId, - entity_type: &str, - set_name: &str, - key: &str, - security_ctx: &SecurityContext, - options: EntityBodyOptions<'_>, + request: EntityBodyRequest<'_>, ) -> Result { - let mut state_json = - load_authorized_entity_body(state, tenant, entity_type, set_name, key, security_ctx) - .await?; + let mut state_json = if let Some(body) = request.authoritative_body { + authorize_read( + state, + tenant, + request.security_ctx, + READ_ACTION, + request.entity_type, + request.key, + &body, + ) + .map_err(|response| *response)?; + body + } else { + load_authorized_entity_body( + state, + tenant, + request.entity_type, + request.set_name, + request.key, + request.security_ctx, + ) + .await? + }; if let Some(obj) = state_json.as_object_mut() { obj.remove("@odata.id"); } - let mut body = annotate_entity(state_json, options.context, options.odata_id); + let mut body = annotate_entity(state_json, request.context, request.odata_id); - if options.enrich { - enrich_entity_response(&mut body, entity_type, set_name, key, state, tenant); + if request.enrich { + enrich_entity_response( + &mut body, + request.entity_type, + request.set_name, + request.key, + state, + tenant, + ); } - if let Some(name) = options.function + if let Some(name) = request.function && let Some(obj) = body.as_object_mut() { obj.insert("@odata.function".to_string(), serde_json::json!(name)); @@ -461,12 +495,12 @@ async fn build_entity_body( apply_entity_query_options( body, - entity_type, + request.entity_type, state, tenant, - security_ctx, - options.query_options, - options.select_before_expand, + request.security_ctx, + request.query_options, + request.select_before_expand, ) .await } @@ -850,9 +884,18 @@ async fn handle_entity( Some(t) => t, None => return entity_set_not_found_response(state, tenant, set_name).await, }; - let key_str = resolve_entity_request_key(state, tenant, &entity_type, key).await; + let resolved = match resolve_entity_request_key(state, tenant, &entity_type, key).await { + Ok(resolved) => resolved, + Err(error) => { + error.record_telemetry(&tracing::Span::current()); + return error.into_response(); + } + }; + let key_str = resolved.entity_id; + let authoritative_body = resolved.authoritative_body; - if state.is_pg_actor_backed(tenant, &entity_type) + if authoritative_body.is_none() + && state.is_pg_actor_backed(tenant, &entity_type) && let Some(actor_sys) = &state.pg_actor_system { let namespace = format!("{tenant}/{key_str}"); @@ -906,11 +949,12 @@ async fn handle_entity( match build_entity_body( state, tenant, - &entity_type, - set_name, - &key_str, - security_ctx, - EntityBodyOptions { + EntityBodyRequest { + entity_type: &entity_type, + set_name, + key: &key_str, + security_ctx, + authoritative_body, context: format!("$metadata#{set_name}/$entity"), odata_id: Some(format!("{set_name}('{key_str}')")), query_options, @@ -1080,11 +1124,12 @@ async fn handle_navigation_entity( match build_entity_body( state, tenant, - &target_type, - &target_set, - &key_str, - security_ctx, - EntityBodyOptions { + EntityBodyRequest { + entity_type: &target_type, + set_name: &target_set, + key: &key_str, + security_ctx, + authoritative_body: None, context: format!("$metadata#{target_set}/$entity"), odata_id: Some(format!("{target_set}('{key_str}')")), query_options, @@ -1140,11 +1185,12 @@ async fn handle_bound_function( match build_entity_body( state, tenant, - &entity_type, - &parent_set, - &parent_key, - security_ctx, - EntityBodyOptions { + EntityBodyRequest { + entity_type: &entity_type, + set_name: &parent_set, + key: &parent_key, + security_ctx, + authoritative_body: None, context: format!("$metadata#{entity_type}"), odata_id: None, query_options, @@ -1411,3 +1457,7 @@ mod next_link_tests { assert_eq!(link.matches("skiptoken").count(), 1); } } + +#[cfg(all(test, feature = "sim"))] +#[path = "read_tests/key_contract.rs"] +mod key_contract_tests; diff --git a/crates/temper-server/src/odata/read_support.rs b/crates/temper-server/src/odata/read_support.rs index 59d6780ec..c547343be 100644 --- a/crates/temper-server/src/odata/read_support.rs +++ b/crates/temper-server/src/odata/read_support.rs @@ -12,11 +12,23 @@ use crate::storage::{ }; mod config; +mod entity_set; +mod journal_materialization; +mod projection_repair; mod select_projection; mod shadow; use config::{catalog_fast_read_enabled, entity_set_materialization_concurrency}; pub(super) use config::{odata_default_page_size, odata_max_entities}; +#[cfg(test)] +use entity_set::select_entity_ids_for_materialization; +pub(super) use entity_set::{record_entity_set_not_found, resolve_entity_set_name}; +use journal_materialization::materialize_entity; +pub(super) use journal_materialization::{ + ExactEntityMaterialization, durable_source_absent_for_catalog_materialization, + materialize_exact_entity, +}; +use projection_repair::remove_deleted_projection; use select_projection::catalog_row_to_selected_entity_body; #[cfg(test)] pub(super) use select_projection::catalog_select_projection_fields; @@ -35,7 +47,7 @@ fn should_read_catalog_for_materialization(prefer_catalog: bool) -> bool { /// can still be synthesized from `status` + `fields` during rolling deploys, /// but those legacy rows do not carry counters, booleans, lists, item counts, /// or fields omitted from the query projection. -fn catalog_row_to_entity_body( +pub(super) fn catalog_row_to_entity_body( entity_type: &str, entity_set_name: &str, row: EntityCatalogRow, @@ -86,26 +98,49 @@ fn catalog_row_to_entity_body( }) } +/// Controls when collection materialization may trust the asynchronous catalog. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum CatalogMaterializationPolicy { + /// Use the catalog directly when a row is available. + Any, + /// Use a catalog row only for the ADR-0077 migration shape with no journal or + /// snapshot. Any durable entity source outranks the asynchronous catalog. + JournalAbsentOnly, +} + +/// An authoritative collection candidate could not be proved current. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum AuthoritativeMaterializationError { + /// The durable journal source or generation could not be stabilized. + JournalUnstable, +} + +impl CatalogMaterializationPolicy { + pub(super) fn uses_catalog(self) -> bool { + true + } +} + async fn try_load_catalog_rows( state: &ServerState, tenant: &TenantId, entity_type: &str, entity_ids: &[String], -) -> BTreeMap { +) -> Result, temper_runtime::persistence::PersistenceError> { let Some(query_plane) = state.query_plane_store() else { - return BTreeMap::new(); + return Ok(BTreeMap::new()); }; match load_catalog_rows_by_id(&query_plane, tenant.as_str(), entity_type, entity_ids).await { - Ok(CatalogRowsLoad::Available(rows)) => rows, - Ok(CatalogRowsLoad::Unsupported) => BTreeMap::new(), + Ok(CatalogRowsLoad::Available(rows)) => Ok(rows), + Ok(CatalogRowsLoad::Unsupported) => Ok(BTreeMap::new()), Err(error) => { tracing::warn!( error = %error, tenant = %tenant, entity_type = %entity_type, - "catalog fast-read failed; falling back to actor materialization" + "catalog fast-read failed" ); - BTreeMap::new() + Err(error) } } } @@ -116,9 +151,9 @@ async fn try_load_selected_catalog_rows( entity_type: &str, entity_ids: &[String], selected_fields: &[String], -) -> BTreeMap { +) -> Result, temper_runtime::persistence::PersistenceError> { let Some(query_plane) = state.query_plane_store() else { - return BTreeMap::new(); + return Ok(BTreeMap::new()); }; match load_selected_catalog_rows_by_id( &query_plane, @@ -129,7 +164,7 @@ async fn try_load_selected_catalog_rows( ) .await { - Ok(CatalogRowsLoad::Available(rows)) => rows, + Ok(CatalogRowsLoad::Available(rows)) => Ok(rows), Ok(CatalogRowsLoad::Unsupported) => { try_load_catalog_rows(state, tenant, entity_type, entity_ids).await } @@ -231,7 +266,9 @@ pub(super) async fn try_load_entity_body_from_catalog( return None; } let ids = [key.to_string()]; - let rows = try_load_catalog_rows(state, tenant, entity_type, &ids).await; + let rows = try_load_catalog_rows(state, tenant, entity_type, &ids) + .await + .ok()?; let row = rows.into_iter().next().map(|(_, r)| r)?; maybe_spawn_catalog_shadow_check(state, tenant, entity_type, &row); let mut body = catalog_row_to_entity_body(entity_type, entity_set_name, row); @@ -245,29 +282,30 @@ pub(super) async fn materialize_entity_set_entities( entity_type: &str, entity_set_name: &str, entity_ids: &[String], - prefer_catalog: bool, + catalog_policy: CatalogMaterializationPolicy, selected_catalog_fields: Option<&[String]>, ) -> MaterializedEntitySet { let selected_catalog_fields_owned = selected_catalog_fields.map(Vec::from); - let mut catalog_hits: BTreeMap = - if should_read_catalog_for_materialization(prefer_catalog) { - match selected_catalog_fields { - Some(select) => { - try_load_selected_catalog_rows(state, tenant, entity_type, entity_ids, select) - .await - } - None => try_load_catalog_rows(state, tenant, entity_type, entity_ids).await, + let catalog_rows = if catalog_policy.uses_catalog() { + match selected_catalog_fields { + Some(select) => { + try_load_selected_catalog_rows(state, tenant, entity_type, entity_ids, select).await } - } else { - BTreeMap::new() - }; + None => try_load_catalog_rows(state, tenant, entity_type, entity_ids).await, + } + } else { + Ok(BTreeMap::new()) + }; + let catalog_unavailable = catalog_rows.is_err(); let mut shadow_budget = CatalogShadowReadBudget::for_entity_set(); + let mut catalog_hits: BTreeMap = catalog_rows.unwrap_or_default(); let concurrency = entity_set_materialization_concurrency(); - let entities = stream::iter(entity_ids.iter().cloned()) + let outcomes = stream::iter(entity_ids.iter().cloned()) .map(|id| { let catalog_row = catalog_hits.remove(&id); - if selected_catalog_fields_owned.is_none() + if catalog_policy == CatalogMaterializationPolicy::Any + && selected_catalog_fields_owned.is_none() && let Some(row) = catalog_row.as_ref() { let _ = maybe_spawn_catalog_shadow_check_with_budget( @@ -285,208 +323,134 @@ pub(super) async fn materialize_entity_set_entities( let selected_catalog_fields = selected_catalog_fields_owned.clone(); async move { if let Some(row) = catalog_row { - let mut entity = match selected_catalog_fields.as_deref() { - Some(select) => catalog_row_to_selected_entity_body( + let catalog_allowed = match catalog_policy { + CatalogMaterializationPolicy::Any => true, + CatalogMaterializationPolicy::JournalAbsentOnly => { + match durable_source_absent_for_catalog_materialization( + &state, + &tenant, + &entity_type, + &id, + ) + .await + { + Ok(absent) => absent, + Err(error) => return Err(error), + } + } + }; + if !catalog_allowed { + return materialize_entity( + &state, + &tenant, &entity_type, &entity_set_name, - row, - select, - ), - None => catalog_row_to_entity_body(&entity_type, &entity_set_name, row), + &id, + catalog_policy, + ) + .await; + } + let catalog_entity = if row.status == "Deleted" { + if catalog_policy == CatalogMaterializationPolicy::Any { + remove_deleted_projection(&state, &tenant, &entity_type, &id).await; + } + None + } else { + let mut entity = match selected_catalog_fields.as_deref() { + Some(select) => catalog_row_to_selected_entity_body( + &entity_type, + &entity_set_name, + row, + select, + ), + None => catalog_row_to_entity_body(&entity_type, &entity_set_name, row), + }; + hydrate_blob_refs_for_tenant(&state, &tenant, &mut entity).await; + Some(entity) }; - hydrate_blob_refs_for_tenant(&state, &tenant, &mut entity).await; - return Some(entity); - } - match state - .get_tenant_entity_state(&tenant, &entity_type, &id) - .await - { - Ok(response) => { - if response.state.status != "Deleted" - && let Some(query_plane) = state.query_plane_store() + if catalog_policy == CatalogMaterializationPolicy::JournalAbsentOnly { + match durable_source_absent_for_catalog_materialization( + &state, + &tenant, + &entity_type, + &id, + ) + .await { - let fields = state.query_projection_fields( - &tenant, - &entity_type, - &response.state.fields, - ); - let projected_state = state.query_projection_state(&response.state); - if let Err(error) = query_plane - .upsert_projection( - tenant.as_str(), + Ok(true) => {} + Ok(false) => { + return materialize_entity( + &state, + &tenant, &entity_type, + &entity_set_name, &id, - &response.state.status, - &fields, - &projected_state, - response.state.sequence_nr, + catalog_policy, ) - .await - { - tracing::debug!( - error = %error, - tenant = %tenant, - entity_type = %entity_type, - entity_id = %id, - "failed to repair query projection after actor materialization fallback" - ); + .await; } + Err(error) => return Err(error), } - let mut entity = serde_json::to_value(&response.state).unwrap_or_default(); - hydrate_blob_refs_for_tenant(&state, &tenant, &mut entity).await; - if let Some(obj) = entity.as_object_mut() { - obj.insert( - "@odata.id".into(), - serde_json::json!(format!("{entity_set_name}('{id}')")), - ); - } - Some(entity) - } - Err(error) => { - tracing::debug!( - error = %error, - tenant = %tenant, - entity_type = %entity_type, - entity_id = %id, - "failed to materialize entity for OData collection" - ); - None } + return Ok(catalog_entity); } + if catalog_unavailable + && catalog_policy == CatalogMaterializationPolicy::JournalAbsentOnly + && durable_source_absent_for_catalog_materialization( + &state, + &tenant, + &entity_type, + &id, + ) + .await? + { + return Err(AuthoritativeMaterializationError::JournalUnstable); + } + materialize_entity( + &state, + &tenant, + &entity_type, + &entity_set_name, + &id, + catalog_policy, + ) + .await } }) .buffered(concurrency) .collect::>() - .await - .into_iter() - .flatten() - .collect::>(); + .await; + let mut entities = Vec::with_capacity(outcomes.len()); + let mut error = None; + for outcome in outcomes { + match outcome { + Ok(Some(entity)) => entities.push(entity), + Ok(None) => {} + Err(materialization_error) => { + error.get_or_insert(materialization_error); + } + } + } MaterializedEntitySet { entities, + error, catalog_shadow_check_budget: shadow_budget.configured(), catalog_shadow_check_scheduled: shadow_budget.scheduled(), } } +/// Materialized collection candidates plus an authoritative-source failure, if any. pub(super) struct MaterializedEntitySet { + /// Successfully materialized live entity bodies. pub(super) entities: Vec, + /// First failure that prevents the partial bodies from being served as complete. + pub(super) error: Option, + /// Configured catalog shadow-read budget for telemetry. pub(super) catalog_shadow_check_budget: usize, + /// Catalog shadow reads actually scheduled for telemetry. pub(super) catalog_shadow_check_scheduled: usize, } -#[cfg(test)] -#[derive(Debug)] -pub(super) struct SelectedEntityIdsForMaterialization { - pub(super) entity_ids: Vec, - pub(super) apply_options: temper_odata::query::types::QueryOptions, - pub(super) precomputed_count: Option, -} - -#[cfg(test)] -#[derive(Debug, Eq, PartialEq)] -pub(super) struct EntitySelectionTooLarge { - pub(super) candidate_count: usize, - pub(super) candidate_budget: usize, -} - -#[cfg(test)] -pub(super) fn select_entity_ids_for_materialization( - mut entity_ids: Vec, - query_options: &temper_odata::query::types::QueryOptions, - default_page_size: usize, - max_entities: usize, - has_row_authorization: bool, -) -> Result { - let has_filter_or_order = - query_options.filter.is_some() || query_options.orderby.is_some() || has_row_authorization; - let mut precomputed_count = None; - - let apply_options = if !has_filter_or_order { - let total_available = entity_ids.len(); - if query_options.count == Some(true) { - precomputed_count = Some(total_available); - } - - let skip = query_options.skip.unwrap_or(0); - let top = query_options.top.unwrap_or(default_page_size); - let requested = top.min(max_entities); - entity_ids = entity_ids - .into_iter() - .skip(skip) - .take(requested) - .collect::>(); - - let mut adjusted = query_options.clone(); - adjusted.skip = None; - adjusted.top = None; - adjusted.count = None; - adjusted - } else { - // When a $filter or $orderby is present, we must materialise ALL - // candidate entities before the filter/sort can be applied. - // Truncating the candidate set before filtering would silently hide - // entities that match the filter but sort past the cutoff — a - // correctness bug that caused system skills to vanish from large File - // collections (see ADR: skill-bootstrap-invisible-in-odata). - // - // Safety cap: impose a hard ceiling (10× max_entities) and reject - // reads that cannot be proven complete inside the candidate budget. - let safety_cap = max_entities.saturating_mul(10); - if entity_ids.len() > safety_cap { - return Err(EntitySelectionTooLarge { - candidate_count: entity_ids.len(), - candidate_budget: safety_cap, - }); - } - - let mut adjusted = query_options.clone(); - if adjusted.top.is_none() { - adjusted.top = Some(default_page_size); - } else if let Some(top) = adjusted.top { - adjusted.top = Some(top.min(max_entities)); - } - adjusted - }; - - Ok(SelectedEntityIdsForMaterialization { - entity_ids, - apply_options, - precomputed_count, - }) -} - -/// Resolve an entity set name from an entity type name. -/// -/// Reverse-lookups the entity_set_map to find the set name for a given type. -pub(super) fn resolve_entity_set_name( - state: &ServerState, - tenant: &TenantId, - entity_type: &str, -) -> String { - let registry = state - .registry - .read() - .expect("registry lock should not be poisoned"); // ci-ok: infallible lock - if let Some(tc) = registry.get_tenant(tenant) { - for (set_name, type_name) in &tc.entity_set_map { - if type_name == entity_type { - return set_name.clone(); - } - } - } - // Fallback: pluralize entity type - format!("{entity_type}s") -} - -/// Record a trajectory entry for an EntitySetNotFound error. -pub(super) async fn record_entity_set_not_found(state: &ServerState, tenant: &str, set_name: &str) { - tracing::warn!(tenant = %tenant, entity_set = %set_name, "entity set not found"); - // Intentionally no trajectory write: read-only operations must not write to the database. - // Previously this wrote a TrajectoryEntry on every failed EntitySetLookup, creating - // unbounded junk rows (4,269 rows, 83% of all trajectories) from phantom entity polling. - let _ = state; // suppress unused warning -} - #[cfg(test)] mod tests; diff --git a/crates/temper-server/src/odata/read_support/entity_set.rs b/crates/temper-server/src/odata/read_support/entity_set.rs new file mode 100644 index 000000000..ae7e7a0bc --- /dev/null +++ b/crates/temper-server/src/odata/read_support/entity_set.rs @@ -0,0 +1,120 @@ +use temper_runtime::tenant::TenantId; + +use crate::state::ServerState; + +#[cfg(test)] +#[derive(Debug)] +pub(super) struct SelectedEntityIdsForMaterialization { + pub(super) entity_ids: Vec, + pub(super) apply_options: temper_odata::query::types::QueryOptions, + pub(super) precomputed_count: Option, +} + +#[cfg(test)] +#[derive(Debug, Eq, PartialEq)] +pub(super) struct EntitySelectionTooLarge { + pub(super) candidate_count: usize, + pub(super) candidate_budget: usize, +} + +#[cfg(test)] +pub(super) fn select_entity_ids_for_materialization( + mut entity_ids: Vec, + query_options: &temper_odata::query::types::QueryOptions, + default_page_size: usize, + max_entities: usize, + has_row_authorization: bool, +) -> Result { + let has_filter_or_order = + query_options.filter.is_some() || query_options.orderby.is_some() || has_row_authorization; + let mut precomputed_count = None; + + let apply_options = if !has_filter_or_order { + let total_available = entity_ids.len(); + if query_options.count == Some(true) { + precomputed_count = Some(total_available); + } + + let skip = query_options.skip.unwrap_or(0); + let top = query_options.top.unwrap_or(default_page_size); + let requested = top.min(max_entities); + entity_ids = entity_ids + .into_iter() + .skip(skip) + .take(requested) + .collect::>(); + + let mut adjusted = query_options.clone(); + adjusted.skip = None; + adjusted.top = None; + adjusted.count = None; + adjusted + } else { + // When a $filter or $orderby is present, we must materialise ALL + // candidate entities before the filter/sort can be applied. + // Truncating the candidate set before filtering would silently hide + // entities that match the filter but sort past the cutoff — a + // correctness bug that caused system skills to vanish from large File + // collections (see ADR: skill-bootstrap-invisible-in-odata). + // + // Safety cap: impose a hard ceiling (10× max_entities) and reject + // reads that cannot be proven complete inside the candidate budget. + let safety_cap = max_entities.saturating_mul(10); + if entity_ids.len() > safety_cap { + return Err(EntitySelectionTooLarge { + candidate_count: entity_ids.len(), + candidate_budget: safety_cap, + }); + } + + let mut adjusted = query_options.clone(); + if adjusted.top.is_none() { + adjusted.top = Some(default_page_size); + } else if let Some(top) = adjusted.top { + adjusted.top = Some(top.min(max_entities)); + } + adjusted + }; + + Ok(SelectedEntityIdsForMaterialization { + entity_ids, + apply_options, + precomputed_count, + }) +} + +/// Resolve an entity set name from an entity type name. +/// +/// Reverse-lookups the entity_set_map to find the set name for a given type. +pub(in crate::odata) fn resolve_entity_set_name( + state: &ServerState, + tenant: &TenantId, + entity_type: &str, +) -> String { + let registry = state + .registry + .read() + .expect("registry lock should not be poisoned"); // ci-ok: infallible lock + if let Some(tc) = registry.get_tenant(tenant) { + for (set_name, type_name) in &tc.entity_set_map { + if type_name == entity_type { + return set_name.clone(); + } + } + } + // Fallback: pluralize entity type + format!("{entity_type}s") +} + +/// Record a trajectory entry for an EntitySetNotFound error. +pub(in crate::odata) async fn record_entity_set_not_found( + state: &ServerState, + tenant: &str, + set_name: &str, +) { + tracing::warn!(tenant = %tenant, entity_set = %set_name, "entity set not found"); + // Intentionally no trajectory write: read-only operations must not write to the database. + // Previously this wrote a TrajectoryEntry on every failed EntitySetLookup, creating + // unbounded junk rows (4,269 rows, 83% of all trajectories) from phantom entity polling. + let _ = state; // suppress unused warning +} diff --git a/crates/temper-server/src/odata/read_support/journal_materialization.rs b/crates/temper-server/src/odata/read_support/journal_materialization.rs new file mode 100644 index 000000000..1d60d24e0 --- /dev/null +++ b/crates/temper-server/src/odata/read_support/journal_materialization.rs @@ -0,0 +1,390 @@ +use super::*; +use crate::entity_actor::{ + EntityRecoveryContext, EntityState, recover_entity_state_from_stable_sources, +}; +use crate::state::source_fenced_projection::repair_projection_from_stable_source; + +const MAX_STABLE_JOURNAL_READ_ATTEMPTS: usize = 3; + +enum EntityMaterializationSource { + Absent, + Actor { + repair_projection: bool, + }, + State { + state: Box, + repair_projection: bool, + }, +} + +/// Result of closing a direct entity read against its durable sources. +pub(in crate::odata) enum ExactEntityMaterialization { + /// A stable journal, snapshot, or actor generation supplied the body. + Present(serde_json::Value), + /// A stable durable generation proves that the entity is terminal. + NotFound, + /// Neither journal nor snapshot exists, so ADR-0077 catalog-only migration + /// compatibility may supply the body. + CatalogCompatible, +} + +/// Prove that neither durable source exists before permitting ADR-0077 catalog +/// compatibility. Journal and snapshot are both read twice so a source transition +/// cannot be flattened into catalog authority. +pub(in crate::odata) async fn durable_source_absent_for_catalog_materialization( + state: &ServerState, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, +) -> Result { + let Some((store, _)) = state.event_journal() else { + return Ok(false); + }; + let persistence_id = format!("{}:{entity_type}:{entity_id}", tenant.as_str()); + for attempt in 1..=MAX_STABLE_JOURNAL_READ_ATTEMPTS { + let before_journal = store.journal_boundary(&persistence_id).await; + let before_snapshot = store.load_snapshot(&persistence_id).await; + let after_journal = store.journal_boundary(&persistence_id).await; + let after_snapshot = store.load_snapshot(&persistence_id).await; + match ( + before_journal, + before_snapshot, + after_journal, + after_snapshot, + ) { + (Ok(before_journal), Ok(before_snapshot), Ok(after_journal), Ok(after_snapshot)) + if before_journal == after_journal && before_snapshot == after_snapshot => + { + return Ok(before_journal.latest_sequence == 0 && before_snapshot.is_none()); + } + (Err(error), _, _, _) + | (_, Err(error), _, _) + | (_, _, Err(error), _) + | (_, _, _, Err(error)) => { + tracing::warn!( + error = %error, + attempt, + tenant = %tenant, + entity_type, + entity_id, + "failed to fence catalog compatibility against durable entity sources" + ); + } + _ => { + tracing::debug!( + attempt, + tenant = %tenant, + entity_type, + entity_id, + "durable entity sources changed while proving catalog compatibility" + ); + } + } + } + Err(AuthoritativeMaterializationError::JournalUnstable) +} + +/// Materialize one direct entity read from its authoritative source before the +/// asynchronous catalog is considered. A terminal durable generation is kept +/// distinct from true source absence so a stale catalog row cannot resurrect it. +pub(in crate::odata) async fn materialize_exact_entity( + state: &ServerState, + tenant: &TenantId, + entity_type: &str, + entity_set_name: &str, + entity_id: &str, +) -> Result { + let source = stable_journal_state(state, tenant, entity_type, entity_id).await?; + let (entity_state, repair_projection) = match source { + EntityMaterializationSource::Absent => { + return Ok(ExactEntityMaterialization::CatalogCompatible); + } + EntityMaterializationSource::Actor { repair_projection } => { + let response = state + .get_tenant_entity_state(tenant, entity_type, entity_id) + .await + .map_err(|error| { + tracing::debug!( + error = %error, + tenant = %tenant, + entity_type, + entity_id, + "failed authoritative actor materialization for direct OData read" + ); + AuthoritativeMaterializationError::JournalUnstable + })?; + (response.state, repair_projection) + } + EntityMaterializationSource::State { + state, + repair_projection, + } => (*state, repair_projection), + }; + + Ok( + match materialize_state( + state, + tenant, + entity_type, + entity_set_name, + entity_id, + entity_state, + repair_projection, + ) + .await + { + Some(body) => ExactEntityMaterialization::Present(body), + None => ExactEntityMaterialization::NotFound, + }, + ) +} + +/// Materialize one collection candidate from the source allowed by `catalog_policy`. +pub(super) async fn materialize_entity( + state: &ServerState, + tenant: &TenantId, + entity_type: &str, + entity_set_name: &str, + entity_id: &str, + catalog_policy: CatalogMaterializationPolicy, +) -> Result, AuthoritativeMaterializationError> { + let source = match catalog_policy { + CatalogMaterializationPolicy::Any => EntityMaterializationSource::Actor { + repair_projection: true, + }, + CatalogMaterializationPolicy::JournalAbsentOnly => { + stable_journal_state(state, tenant, entity_type, entity_id).await? + } + }; + let (entity_state, repair_projection) = match source { + EntityMaterializationSource::Absent => return Ok(None), + EntityMaterializationSource::Actor { repair_projection } => match state + .get_tenant_entity_state(tenant, entity_type, entity_id) + .await + { + Ok(response) => (response.state, repair_projection), + Err(error) => { + tracing::debug!( + error = %error, + tenant = %tenant, + entity_type, + entity_id, + "failed to materialize entity for OData collection" + ); + return match catalog_policy { + CatalogMaterializationPolicy::Any => Ok(None), + CatalogMaterializationPolicy::JournalAbsentOnly => { + Err(AuthoritativeMaterializationError::JournalUnstable) + } + }; + } + }, + EntityMaterializationSource::State { + state, + repair_projection, + } => (*state, repair_projection), + }; + + Ok(materialize_state( + state, + tenant, + entity_type, + entity_set_name, + entity_id, + entity_state, + repair_projection, + ) + .await) +} + +async fn stable_journal_state( + state: &ServerState, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, +) -> Result { + let Some((store, backend)) = state.event_journal() else { + return Ok(EntityMaterializationSource::Actor { + repair_projection: true, + }); + }; + let persistence_id = format!("{}:{entity_type}:{entity_id}", tenant.as_str()); + let table = { + let registry = state.registry.read().expect("registry lock poisoned"); + registry + .get_table_live(tenant, entity_type) + .map(|table| table.read().expect("table lock poisoned").clone()) + } + .or_else(|| { + state + .transition_tables + .get(entity_type) + .map(|table| (**table).clone()) + }); + let Some(table) = table else { + tracing::warn!( + tenant = %tenant, + entity_type, + entity_id, + "missing transition table for exact-key journal materialization" + ); + return Err(AuthoritativeMaterializationError::JournalUnstable); + }; + let blob_store = state.blob_store_for_tenant(tenant).ok(); + + for attempt in 1..=MAX_STABLE_JOURNAL_READ_ATTEMPTS { + let source = match recover_entity_state_from_stable_sources(EntityRecoveryContext { + tenant: tenant.as_str(), + entity_type, + entity_id, + table: &table, + store: &store, + backend, + initial_fields: &serde_json::json!({}), + blob_store: blob_store.as_ref(), + }) + .await + { + Ok(recovered) => recovered, + Err(error) => { + tracing::warn!( + error = %error, + attempt, + tenant = %tenant, + entity_type, + entity_id, + "failed strict replay for exact-key fallback materialization" + ); + continue; + } + }; + if source.state.is_none() { + return Ok(EntityMaterializationSource::Absent); + } + if source.journal_sequence == 0 { + return Ok(EntityMaterializationSource::State { + state: Box::new( + source + .state + .expect("snapshot-only source was validated as present"), + ), + repair_projection: false, + }); + } + match repair_projection_from_stable_source( + state, + tenant, + entity_type, + entity_id, + &store, + &persistence_id, + &source, + ) + .await + { + Ok(true) => { + return Ok(EntityMaterializationSource::State { + state: Box::new( + source + .state + .expect("journal source was validated as present"), + ), + // Repair and its closing source fence completed together. + // Repeating it after this return would reopen the race. + repair_projection: false, + }); + } + Ok(false) => {} + Err(error) => { + tracing::warn!( + error = %error, + attempt, + tenant = %tenant, + entity_type, + entity_id, + "source-fenced exact-key projection repair failed" + ); + continue; + } + } + tracing::debug!( + attempt, + tenant = %tenant, + entity_type, + entity_id, + "durable source advanced during exact-key fallback materialization; retrying" + ); + } + + tracing::warn!( + attempts = MAX_STABLE_JOURNAL_READ_ATTEMPTS, + tenant = %tenant, + entity_type, + entity_id, + "exact-key fallback materialization could not obtain a stable journal generation" + ); + Err(AuthoritativeMaterializationError::JournalUnstable) +} + +async fn materialize_state( + state: &ServerState, + tenant: &TenantId, + entity_type: &str, + entity_set_name: &str, + entity_id: &str, + entity_state: EntityState, + repair_projection: bool, +) -> Option { + if repair_projection { + repair_projection_for_state(state, tenant, entity_type, entity_id, &entity_state).await; + } + if entity_state.status == "Deleted" { + return None; + } + let mut entity = serde_json::to_value(entity_state).unwrap_or_default(); + hydrate_blob_refs_for_tenant(state, tenant, &mut entity).await; + if let Some(object) = entity.as_object_mut() { + object.insert( + "@odata.id".into(), + serde_json::json!(format!("{entity_set_name}('{entity_id}')")), + ); + } + Some(entity) +} + +async fn repair_projection_for_state( + state: &ServerState, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + entity_state: &EntityState, +) { + if entity_state.status == "Deleted" { + remove_deleted_projection(state, tenant, entity_type, entity_id).await; + return; + } + if let Some(query_plane) = state.query_plane_store() { + let fields = state.query_projection_fields(tenant, entity_type, &entity_state.fields); + let projected_state = state.query_projection_state(entity_state); + if let Err(error) = query_plane + .upsert_projection( + tenant.as_str(), + entity_type, + entity_id, + &entity_state.status, + &fields, + &projected_state, + entity_state.sequence_nr, + ) + .await + { + tracing::debug!( + error = %error, + tenant = %tenant, + entity_type, + entity_id, + "failed to repair query projection after authoritative materialization fallback" + ); + } + } +} diff --git a/crates/temper-server/src/odata/read_support/projection_repair.rs b/crates/temper-server/src/odata/read_support/projection_repair.rs new file mode 100644 index 000000000..d8745dd14 --- /dev/null +++ b/crates/temper-server/src/odata/read_support/projection_repair.rs @@ -0,0 +1,28 @@ +//! Idempotent query-projection repairs discovered during authoritative reads. + +use temper_runtime::tenant::TenantId; + +use crate::state::ServerState; + +pub(super) async fn remove_deleted_projection( + state: &ServerState, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, +) { + let Some(query_plane) = state.query_plane_store() else { + return; + }; + if let Err(error) = query_plane + .remove_projection(tenant.as_str(), entity_type, entity_id) + .await + { + tracing::debug!( + error = %error, + tenant = %tenant, + entity_type, + entity_id, + "failed to repair deleted query projection during OData materialization" + ); + } +} diff --git a/crates/temper-server/src/odata/read_tests/key_contract.rs b/crates/temper-server/src/odata/read_tests/key_contract.rs new file mode 100644 index 000000000..5ea8e9992 --- /dev/null +++ b/crates/temper-server/src/odata/read_tests/key_contract.rs @@ -0,0 +1,489 @@ +//! Composite-key URL fallback must remain authoritative before index coverage. + +use std::sync::Arc; + +use super::*; +use crate::registry::SpecRegistry; +use crate::storage::{ + BackendLabel, BoxedEventStore, EntityCatalogRow, QueryPlaneStore, QueryProjectionFieldsRow, + StorageStack, +}; +use async_trait::async_trait; +use temper_runtime::ActorSystem; +use temper_runtime::persistence::{ + EntityKeyRow, EventMetadata, EventStore, IndexReconciliation, PersistenceAppend, + PersistenceEnvelope, +}; +use temper_runtime::scheduler::{install_deterministic_context, sim_now, sim_uuid}; +use temper_spec::csdl::parse_csdl; +use temper_store_sim::SimEventStore; + +const CSDL_XML: &str = include_str!("../../../../../test-fixtures/specs/model.csdl.xml"); +const ORDER_IOA: &str = include_str!("../../../../../test-fixtures/specs/order.ioa.toml"); + +struct StaleCatalogQueryPlane { + row: EntityCatalogRow, +} + +#[async_trait] +impl QueryPlaneStore for StaleCatalogQueryPlane { + async fn upsert_projection( + &self, + _tenant: &str, + _entity_type: &str, + _entity_id: &str, + _status: &str, + _fields: &serde_json::Value, + _state: &serde_json::Value, + _sequence_nr: u64, + ) -> Result<(), temper_runtime::persistence::PersistenceError> { + Ok(()) + } + + async fn remove_projection( + &self, + _tenant: &str, + _entity_type: &str, + _entity_id: &str, + ) -> Result<(), temper_runtime::persistence::PersistenceError> { + Ok(()) + } + + async fn query_field_index( + &self, + _tenant: &str, + _entity_type: &str, + _where_clause: &str, + _params: Vec, + ) -> Result>, temper_runtime::persistence::PersistenceError> { + Ok(None) + } + + async fn load_projection_fields_many( + &self, + _tenant: &str, + _entity_type: &str, + _entity_ids: &[String], + _field_names: &[&str], + ) -> Result>, temper_runtime::persistence::PersistenceError> + { + Ok(None) + } + + async fn load_entity_catalog_rows( + &self, + _tenant: &str, + entity_type: &str, + entity_ids: &[String], + ) -> Result>, temper_runtime::persistence::PersistenceError> { + let rows = if entity_type == "Order" && entity_ids.contains(&self.row.entity_id) { + vec![self.row.clone()] + } else { + Vec::new() + }; + Ok(Some(rows)) + } + + async fn projected_entity_counts_by_tenant( + &self, + ) -> Result>, temper_runtime::persistence::PersistenceError> { + Ok(None) + } +} + +/// A pre-coverage entity may have durable key fields but no key row. Composite +/// URL resolution must scan replayable state and preserve the working lookup, +/// never delegate ownership to the asynchronous field projection. +#[tokio::test] +async fn composite_key_without_coverage_resolves_from_authoritative_state() { + let (_guard, _clock, _ids) = install_deterministic_context(247); + let tenant = TenantId::default(); + let entity_id = "ord-composite-pre-coverage"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let workspace = "ws-composite"; + let path = "/pre-coverage"; + let store = SimEventStore::no_faults(247); + let timestamp = sim_now(); + EventStore::append( + &store, + &persistence_id, + 0, + &[PersistenceEnvelope { + sequence_nr: 0, + event_type: "Create".to_string(), + payload: serde_json::json!({ + "action": "Create", + "from_status": "Draft", + "to_status": "Draft", + "timestamp": timestamp, + "params": {"WorkspaceId": workspace, "Path": path}, + "idempotency_key": null, + }), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: persistence_id.clone(), + }, + }], + ) + .await + .expect("seed legacy stream without a key row"); + EventStore::save_snapshot( + &store, + &persistence_id, + 1, + &serde_json::to_vec(&serde_json::json!({ + "entity_type": "Order", + "entity_id": entity_id, + "status": "Draft", + "item_count": 0, + "fields": { + "Id": entity_id, + "Status": "Draft", + "WorkspaceId": workspace, + "Path": path, + }, + })) + .expect("snapshot serialization"), + ) + .await + .expect("seed replayable key fields"); + + let csdl = parse_csdl(CSDL_XML).expect("CSDL parse"); + let mut registry = SpecRegistry::new(); + registry.register_tenant( + tenant.as_str(), + csdl, + CSDL_XML.to_string(), + &[("Order", ORDER_IOA)], + ); + let mut state = + ServerState::from_registry(ActorSystem::new("composite-key-pre-coverage"), registry); + state.set_storage_stack(StorageStack::from_sim(store.clone(), None)); + + assert!( + EventStore::lookup_by_key( + &store, + tenant.as_str(), + "Order", + "ws_path", + &crate::key_index::canonical_key_hash( + "ws_path", + &["WorkspaceId".to_string(), "Path".to_string()], + serde_json::json!({"WorkspaceId": workspace, "Path": path}) + .as_object() + .expect("key fields"), + ) + .expect("complete key"), + ) + .await + .expect("key lookup") + .is_none(), + "precondition: the legacy stream has no declared-key row" + ); + + let resolved = try_resolve_composite_entity_key( + &state, + &tenant, + "Order", + &[ + ("WorkspaceId".to_string(), workspace.to_string()), + ("Path".to_string(), path.to_string()), + ], + ) + .await; + let resolved = match resolved { + Ok(resolved) => resolved, + Err(_) => panic!("pre-coverage composite lookup must remain within budget"), + }; + assert_eq!( + resolved + .as_ref() + .map(|resolved| resolved.entity_id.as_str()), + Some(entity_id) + ); +} + +/// Even after key coverage is complete, the catalog is not the body authority. +/// A covered hit identifies the entity, then composite-key GET materializes its +/// current journal/snapshot state so a lagging pre-rename catalog row cannot leak. +#[tokio::test] +async fn covered_composite_hit_materializes_authoritative_body_not_stale_catalog() { + let (_guard, _clock, _ids) = install_deterministic_context(248); + let tenant = TenantId::default(); + let entity_id = "ord-composite-covered"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let workspace = "ws-composite"; + let current_path = "/current"; + let stale_path = "/stale"; + let store = SimEventStore::no_faults(248); + let timestamp = sim_now(); + let current_fields = serde_json::json!({ + "Id": entity_id, + "Status": "Draft", + "WorkspaceId": workspace, + "Path": current_path, + }); + let current_state = serde_json::json!({ + "entity_type": "Order", + "entity_id": entity_id, + "status": "Draft", + "item_count": 0, + "fields": current_fields, + }); + let signature = crate::key_index::declared_key_set_signature( + &temper_jit::table::TransitionTable::from_ioa_source(ORDER_IOA).keys, + ); + EventStore::append_with_index_rows( + &store, + &persistence_id, + 0, + &[PersistenceEnvelope { + sequence_nr: 0, + event_type: "Create".to_string(), + payload: serde_json::json!({ + "action": "Create", + "from_status": "Draft", + "to_status": "Draft", + "timestamp": timestamp, + "params": {"WorkspaceId": workspace, "Path": current_path}, + "idempotency_key": null, + }), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: persistence_id.clone(), + }, + }], + &[EntityKeyRow { + key_name: "ws_path".to_string(), + key_hash: crate::key_index::canonical_key_hash( + "ws_path", + &["WorkspaceId".to_string(), "Path".to_string()], + current_fields.as_object().expect("current fields"), + ) + .expect("complete current key"), + }], + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(signature.clone()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .expect("seed authoritative key owner"); + EventStore::save_snapshot( + &store, + &persistence_id, + 1, + &serde_json::to_vec(¤t_state).expect("snapshot serialization"), + ) + .await + .expect("seed current authoritative state"); + EventStore::mark_key_index_backfilled(&store, tenant.as_str(), "Order", &signature) + .await + .expect("mark current coverage"); + + let stale_fields = serde_json::json!({ + "Id": entity_id, + "Status": "Draft", + "WorkspaceId": workspace, + "Path": stale_path, + }); + let stale_state = serde_json::json!({ + "entity_type": "Order", + "entity_id": entity_id, + "status": "Draft", + "fields": stale_fields, + "sequence_nr": 0, + "events": [], + }); + let query_plane = Arc::new(StaleCatalogQueryPlane { + row: EntityCatalogRow { + entity_id: entity_id.to_string(), + status: "Draft".to_string(), + fields: stale_state["fields"].clone(), + state: Some(stale_state), + sequence_nr: 0, + }, + }); + + let csdl = parse_csdl(CSDL_XML).expect("CSDL parse"); + let mut registry = SpecRegistry::new(); + registry.register_tenant( + tenant.as_str(), + csdl, + CSDL_XML.to_string(), + &[("Order", ORDER_IOA)], + ); + let mut state = + ServerState::from_registry(ActorSystem::new("composite-key-covered-body"), registry); + state.set_storage_stack(StorageStack::new( + BackendLabel::Sim, + BoxedEventStore::new(store), + None, + None, + None, + None, + Some(query_plane as Arc), + None, + None, + None, + )); + + let security_ctx = SecurityContext::system(); + let composite_key = KeyValue::Composite(vec![ + ("WorkspaceId".to_string(), workspace.to_string()), + ("Path".to_string(), current_path.to_string()), + ]); + let resolved = match resolve_entity_request_key(&state, &tenant, "Order", &composite_key).await + { + Ok(resolved) => resolved, + Err(_) => panic!("covered key hit must resolve"), + }; + assert_eq!(resolved.entity_id, entity_id); + let query_options = QueryOptions::default(); + let body = build_entity_body( + &state, + &tenant, + EntityBodyRequest { + entity_type: "Order", + set_name: "Orders", + key: &resolved.entity_id, + security_ctx: &security_ctx, + authoritative_body: resolved.authoritative_body, + context: "$metadata#Orders/$entity".to_string(), + odata_id: None, + query_options: &query_options, + enrich: false, + function: None, + select_before_expand: false, + }, + ) + .await; + let body = match body { + Ok(body) => body, + Err(_) => panic!("covered composite hit must load authoritative body"), + }; + assert_eq!(body["fields"]["Path"], current_path); + assert_ne!(body["fields"]["Path"], stale_path); + + // A composite-key URL is still an entity GET. Its internal identity lookup + // must not add the collection-level `list` permission that direct-ID GET does + // not require; the final body remains protected by entity `read`. + state + .authz + .reload_tenant_policies( + tenant.as_str(), + r#" + permit(principal, action == Action::"read", resource is Order); + forbid(principal, action == Action::"list", resource is Order); + "#, + ) + .expect("install read-without-list policy"); + let reader = SecurityContext::from_headers(&[( + "x-temper-principal-id".to_string(), + "composite-reader".to_string(), + )]); + assert!( + state + .authorize_with_context( + &reader, + "list", + "Order", + &std::collections::BTreeMap::new(), + tenant.as_str(), + ) + .is_err(), + "precondition: the reader has no collection list permission" + ); + let response = handle_entity( + &state, + &tenant, + &reader, + "Orders", + &composite_key, + &QueryOptions::default(), + ) + .await; + assert_eq!( + response.status(), + StatusCode::OK, + "composite GET must retain entity read-without-list semantics" + ); +} + +/// An incomplete key index over the bounded scan budget must preserve the +/// query-plane 413 instead of degrading to a raw composite string and 404. +#[tokio::test] +async fn oversized_incomplete_composite_lookup_returns_query_too_large() { + let (_guard, _clock, _ids) = install_deterministic_context(249); + let tenant = TenantId::default(); + let store = SimEventStore::no_faults(249); + // The paging wrapper gives the one-row lookahead a ten-candidate headroom + // (`max_entities + 1`, multiplied by the scan factor). Exceed that effective + // budget so this exercises the HTTP 413 mapping rather than a bounded miss. + let scan_budget = QueryPlaneReadBudget::from_config() + .candidate_budget() + .saturating_add(10); + let timestamp = sim_now(); + let event = PersistenceEnvelope { + sequence_nr: 0, + event_type: "LegacyCreate".to_string(), + payload: serde_json::json!({}), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: "composite-budget-seed".to_string(), + }, + }; + let appends = (0..=scan_budget) + .map(|index| PersistenceAppend { + persistence_id: format!("{tenant}:Order:ord-budget-{index:05}"), + expected_sequence: 0, + events: vec![event.clone()], + key_rows: Vec::new(), + reconcile_keys: false, + key_set_signature: None, + snapshot_source: Default::default(), + batch_idempotency: None, + }) + .collect::>(); + EventStore::append_batch(&store, &appends) + .await + .expect("seed over-budget legacy streams"); + + let csdl = parse_csdl(CSDL_XML).expect("CSDL parse"); + let mut registry = SpecRegistry::new(); + registry.register_tenant( + tenant.as_str(), + csdl, + CSDL_XML.to_string(), + &[("Order", ORDER_IOA)], + ); + let mut state = + ServerState::from_registry(ActorSystem::new("composite-key-over-budget"), registry); + state.set_storage_stack(StorageStack::from_sim(store, None)); + let key = KeyValue::Composite(vec![ + ("WorkspaceId".to_string(), "missing-workspace".to_string()), + ("Path".to_string(), "/missing".to_string()), + ]); + let response = handle_entity( + &state, + &tenant, + &SecurityContext::system(), + "Orders", + &key, + &QueryOptions::default(), + ) + .await; + assert_eq!(response.status(), StatusCode::PAYLOAD_TOO_LARGE); +} diff --git a/crates/temper-server/src/odata/write.rs b/crates/temper-server/src/odata/write.rs index d7e610c95..83e3df137 100644 --- a/crates/temper-server/src/odata/write.rs +++ b/crates/temper-server/src/odata/write.rs @@ -32,8 +32,8 @@ use crate::blobs::hydrate_blob_refs_for_tenant; use crate::identity::ResolvedIdentity; use crate::request_context::{AgentContext, extract_agent_context, remote_parent_context}; use crate::response::{ODataResponse, odata_error}; -use crate::state::ServerState; use crate::state::trajectory::{TrajectoryEntry, TrajectorySource}; +use crate::state::{ServerState, TenantGenerationLease}; type ODataWriteError = Box; @@ -223,6 +223,7 @@ async fn authorize_existing_mutation( pub async fn handle_odata_post( State(state): State, resolved_id: Option>, + generation_lease: Option>, headers: HeaderMap, axum::extract::Path(path): axum::extract::Path, Query(query_params): Query>, @@ -237,11 +238,15 @@ pub async fn handle_odata_post( tracing::Span::current().set_parent(remote_parent); } let resolved_identity = resolved_id.map(|Extension(id)| id); + let generation_lease = generation_lease.map(|Extension(lease)| lease); // Enrich agent context with credential-resolved identity (ADR-0033). if let Some(ref identity) = resolved_identity { agent_ctx.agent_id = Some(identity.agent_instance_id.clone()); agent_ctx.agent_type = Some(identity.agent_type_name.clone()); } + if let Some(lease) = generation_lease.as_ref() { + agent_ctx = agent_ctx.with_tenant_generation_lease(lease.clone()); + } let await_integration = query_params .get("await_integration") .map(|v| v == "true") @@ -688,6 +693,7 @@ pub async fn handle_odata_post( await_integration, idempotency_key.clone(), resolved_identity.as_ref(), + generation_lease.as_ref(), ) .await } @@ -839,7 +845,14 @@ pub async fn handle_odata_patch( } match state - .update_tenant_entity_fields(&tenant, &entity_type, &key_str, body_json, false) + .update_tenant_entity_fields( + &tenant, + &entity_type, + &key_str, + body_json, + false, + agent_ctx.idempotency_key.clone(), + ) .await { Ok(response) => { @@ -988,7 +1001,14 @@ pub async fn handle_odata_put( } match state - .update_tenant_entity_fields(&tenant, &entity_type, &key_str, body_json, true) + .update_tenant_entity_fields( + &tenant, + &entity_type, + &key_str, + body_json, + true, + agent_ctx.idempotency_key.clone(), + ) .await { Ok(response) => { diff --git a/crates/temper-server/src/platform_store.rs b/crates/temper-server/src/platform_store.rs index 6fb57ed66..a29cdcdb0 100644 --- a/crates/temper-server/src/platform_store.rs +++ b/crates/temper-server/src/platform_store.rs @@ -8,6 +8,12 @@ use std::collections::BTreeMap; +#[cfg(feature = "sim")] +mod sim; + +#[cfg(feature = "sim")] +pub use sim::*; + // --------------------------------------------------------------------------- // Row / update types // --------------------------------------------------------------------------- @@ -29,6 +35,87 @@ pub struct SpecRow { pub committed: bool, } +/// One spec row in an atomic durable publication. +#[derive(Debug, Clone, Copy)] +pub struct SpecPublication<'a> { + /// Entity type owning this specification. + pub entity_type: &'a str, + /// IOA source published for the entity type. + pub ioa_source: &'a str, + /// Tenant CSDL generation paired with this publication. + pub csdl_xml: &'a str, + /// Stable digest of the IOA source. + pub content_hash: &'a str, +} + +/// Durable omission semantics for an atomic spec publication. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum SpecPublicationMode { + /// Upsert submitted rows and preserve every omitted durable spec. + Merge, + /// Make the submitted entity types the tenant's complete durable set. + Replace, +} + +/// Tenant-constraint mutation included in an atomic spec publication. +#[derive(Debug, Clone, Copy)] +pub enum TenantConstraintsPublication<'a> { + /// Leave the durable tenant constraint row unchanged. + Preserve, + /// Replace the row, or delete it when the payload is `None`. + Replace(Option<&'a str>), +} + +/// Tenant Cedar-policy mutation included in an atomic spec publication. +#[derive(Debug, Clone, Copy)] +pub enum TenantPolicyPublication<'a> { + /// Leave the durable tenant policy unchanged. + Preserve, + /// Replace the durable tenant policy with this complete policy set. + Replace(&'a str), +} + +/// OS-app metadata committed in the same transaction as its spec generation. +#[derive(Debug, Clone, Copy)] +pub struct OsAppPublication<'a> { + /// Complete target digest metadata and in-progress publication context. + pub record: &'a InstalledAppRecord, + /// Policy owner whose complete row set is replaced by this app generation. + pub policy_owner: Option<&'a str>, + /// Granular policy rows committed beside the aggregate tenant policy. + pub policy_entries: &'a [PolicyEntryPublication<'a>], +} + +/// Complete granular Cedar row generation replaced atomically with specs. +#[derive(Debug, Clone, Copy)] +pub struct PolicyGenerationPublication<'a> { + /// Stable owner whose prior row set is replaced. + pub policy_owner: &'a str, + /// Complete next row set owned by `policy_owner`. + pub policy_entries: &'a [PolicyEntryPublication<'a>], +} + +/// One granular Cedar policy row in an atomic OS-app publication. +#[derive(Debug, Clone, Copy)] +pub struct PolicyEntryPublication<'a> { + pub policy_id: &'a str, + pub cedar_text: &'a str, + pub created_by: &'a str, +} + +/// WASM module bytes committed in the same durable generation as specs. +#[derive(Debug, Clone, Copy)] +pub struct WasmPublication<'a> { + /// Tenant-local module name. + pub module_name: &'a str, + /// Validated module bytes. + pub wasm_bytes: &'a [u8], + /// SHA-256 digest of `wasm_bytes`. + pub sha256_hash: &'a str, + /// Durable provenance (`upload` or `bundled`). + pub source: &'a str, +} + /// Update payload for [`PlatformStore::persist_spec_verification()`]. #[derive(Debug, Clone)] pub struct SpecVerificationUpdate<'a> { @@ -67,6 +154,7 @@ pub struct PolicyEntryRow { pub tenant: String, pub policy_id: String, pub cedar_text: String, + pub created_by: String, pub enabled: bool, } @@ -119,6 +207,24 @@ pub trait PlatformStore: Send + Sync { content_hash: &str, ) -> Result<(), String>; + /// Atomically publish spec rows and return every durable type removed by a + /// replace-mode registration. + #[expect( + clippy::too_many_arguments, + reason = "atomic publication carries every durable tenant-generation component" + )] + async fn publish_specs( + &self, + tenant: &str, + specs: &[SpecPublication<'_>], + mode: SpecPublicationMode, + constraints: TenantConstraintsPublication<'_>, + policy: TenantPolicyPublication<'_>, + os_app: Option>, + policy_generation: Option>, + wasm_modules: &[WasmPublication<'_>], + ) -> Result, String>; + /// Load all persisted specs (for startup recovery). async fn load_specs(&self) -> Result, String>; @@ -251,6 +357,104 @@ impl PlatformStore for TursoEventStore { .map_err(|e| e.to_string()) } + async fn publish_specs( + &self, + tenant: &str, + specs: &[SpecPublication<'_>], + mode: SpecPublicationMode, + constraints: TenantConstraintsPublication<'_>, + policy: TenantPolicyPublication<'_>, + os_app: Option>, + policy_generation: Option>, + wasm_modules: &[WasmPublication<'_>], + ) -> Result, String> { + let rows = specs + .iter() + .map(|spec| { + ( + spec.entity_type, + spec.ioa_source, + spec.csdl_xml, + spec.content_hash, + ) + }) + .collect::>(); + let constraints = match constraints { + TenantConstraintsPublication::Preserve => None, + TenantConstraintsPublication::Replace(source) => Some(source), + }; + let policy = match policy { + TenantPolicyPublication::Preserve => None, + TenantPolicyPublication::Replace(source) => Some(source), + }; + let granular_policy = policy_generation + .map(|publication| (publication.policy_owner, publication.policy_entries)) + .or_else(|| { + os_app.and_then(|publication| { + publication + .policy_owner + .map(|owner| (owner, publication.policy_entries)) + }) + }); + let policy_owner = granular_policy.map(|(owner, _)| owner); + let policy_entries = granular_policy + .map(|(_, entries)| { + entries + .iter() + .map(|entry| (entry.policy_id, entry.cedar_text, entry.created_by)) + .collect::>() + }) + .unwrap_or_default(); + let os_app = os_app.map(|publication| TursoInstalledAppRow { + tenant_id: publication.record.tenant.clone(), + app_name: publication.record.app_name.clone(), + source_kind: publication.record.source_kind.clone(), + app_ref: publication.record.app_ref.clone(), + version_hash: publication.record.version_hash.clone(), + pinned_version_hash: publication.record.pinned_version_hash.clone(), + current_version_hash: publication.record.current_version_hash.clone(), + follow_policy: publication.record.follow_policy.clone(), + closure_id: publication.record.closure_id.clone(), + registry_url: publication.record.registry_url.clone(), + registry_tenant: publication.record.registry_tenant.clone(), + app_version: publication.record.app_version.clone(), + bundle_digest: publication.record.bundle_digest.clone(), + spec_digest: publication.record.spec_digest.clone(), + policy_digest: publication.record.policy_digest.clone(), + wasm_digest: publication.record.wasm_digest.clone(), + content_digest: publication.record.content_digest.clone(), + seed_digest: publication.record.seed_digest.clone(), + installed_at: publication.record.installed_at.clone().unwrap_or_default(), + last_reconciled_at: publication.record.last_reconciled_at.clone(), + status: publication.record.status.clone(), + }); + let wasm_rows = wasm_modules + .iter() + .map(|module| { + ( + module.module_name, + module.wasm_bytes, + module.sha256_hash, + module.source, + ) + }) + .collect::>(); + TursoEventStore::publish_specs( + self, + tenant, + &rows, + mode == SpecPublicationMode::Replace, + constraints, + policy, + os_app.as_ref(), + &wasm_rows, + policy_owner, + &policy_entries, + ) + .await + .map_err(|e| e.to_string()) + } + async fn load_specs(&self) -> Result, String> { let rows = self.load_specs().await.map_err(|e| e.to_string())?; Ok(rows @@ -336,6 +540,7 @@ impl PlatformStore for TursoEventStore { tenant: row.tenant, policy_id: row.policy_id, cedar_text: row.cedar_text, + created_by: row.created_by, enabled: row.enabled, }) .collect()) @@ -507,6 +712,104 @@ impl PlatformStore for PostgresEventStore { .map_err(|e| e.to_string()) } + async fn publish_specs( + &self, + tenant: &str, + specs: &[SpecPublication<'_>], + mode: SpecPublicationMode, + constraints: TenantConstraintsPublication<'_>, + policy: TenantPolicyPublication<'_>, + os_app: Option>, + policy_generation: Option>, + wasm_modules: &[WasmPublication<'_>], + ) -> Result, String> { + let rows = specs + .iter() + .map(|spec| { + ( + spec.entity_type, + spec.ioa_source, + spec.csdl_xml, + spec.content_hash, + ) + }) + .collect::>(); + let constraints = match constraints { + TenantConstraintsPublication::Preserve => None, + TenantConstraintsPublication::Replace(source) => Some(source), + }; + let policy = match policy { + TenantPolicyPublication::Preserve => None, + TenantPolicyPublication::Replace(source) => Some(source), + }; + let granular_policy = policy_generation + .map(|publication| (publication.policy_owner, publication.policy_entries)) + .or_else(|| { + os_app.and_then(|publication| { + publication + .policy_owner + .map(|owner| (owner, publication.policy_entries)) + }) + }); + let policy_owner = granular_policy.map(|(owner, _)| owner); + let policy_entries = granular_policy + .map(|(_, entries)| { + entries + .iter() + .map(|entry| (entry.policy_id, entry.cedar_text, entry.created_by)) + .collect::>() + }) + .unwrap_or_default(); + let os_app = os_app.map(|publication| PostgresInstalledAppRow { + tenant: publication.record.tenant.clone(), + app_name: publication.record.app_name.clone(), + source_kind: publication.record.source_kind.clone(), + app_ref: publication.record.app_ref.clone(), + version_hash: publication.record.version_hash.clone(), + pinned_version_hash: publication.record.pinned_version_hash.clone(), + current_version_hash: publication.record.current_version_hash.clone(), + follow_policy: publication.record.follow_policy.clone(), + closure_id: publication.record.closure_id.clone(), + registry_url: publication.record.registry_url.clone(), + registry_tenant: publication.record.registry_tenant.clone(), + app_version: publication.record.app_version.clone(), + bundle_digest: publication.record.bundle_digest.clone(), + spec_digest: publication.record.spec_digest.clone(), + policy_digest: publication.record.policy_digest.clone(), + wasm_digest: publication.record.wasm_digest.clone(), + content_digest: publication.record.content_digest.clone(), + seed_digest: publication.record.seed_digest.clone(), + installed_at: publication.record.installed_at.clone().unwrap_or_default(), + last_reconciled_at: publication.record.last_reconciled_at.clone(), + status: publication.record.status.clone(), + }); + let wasm_rows = wasm_modules + .iter() + .map(|module| { + ( + module.module_name, + module.wasm_bytes, + module.sha256_hash, + module.source, + ) + }) + .collect::>(); + PostgresEventStore::publish_specs( + self, + tenant, + &rows, + mode == SpecPublicationMode::Replace, + constraints, + policy, + os_app.as_ref(), + &wasm_rows, + policy_owner, + &policy_entries, + ) + .await + .map_err(|e| e.to_string()) + } + async fn load_specs(&self) -> Result, String> { let rows = self.load_specs().await.map_err(|e| e.to_string())?; Ok(rows @@ -596,6 +899,7 @@ impl PlatformStore for PostgresEventStore { tenant: row.tenant, policy_id: row.policy_id, cedar_text: row.cedar_text, + created_by: row.created_by, enabled: row.enabled, }) .collect()) @@ -751,546 +1055,3 @@ impl PlatformStore for PostgresEventStore { .map_err(|e| e.to_string()) } } - -// --------------------------------------------------------------------------- -// SimPlatformStore (behind cfg(feature = "sim")) -// --------------------------------------------------------------------------- - -#[cfg(feature = "sim")] -pub use sim_platform_store::*; - -#[cfg(feature = "sim")] -mod sim_platform_store { - use super::*; - use std::collections::{BTreeMap, BTreeSet}; - use std::sync::{Arc, Mutex}; - use temper_store_sim::DeterministicRng; - - /// Fault injection configuration for platform store simulation. - /// - /// Controls the probability of injected failures during platform store - /// operations. All probabilities are in \[0.0, 1.0\]. - #[derive(Debug, Clone)] - pub struct SimPlatformFaultConfig { - /// Probability of a write failure on spec upsert. - pub spec_write_failure_prob: f64, - /// Probability of a read failure on spec load. - pub spec_read_failure_prob: f64, - /// Probability of a write failure on policy upsert. - pub policy_write_failure_prob: f64, - /// Probability of a read failure on policy load. - pub policy_read_failure_prob: f64, - /// Probability of a failure recording an installed app. - pub app_record_failure_prob: f64, - /// Probability of a failure listing installed apps. - pub app_list_failure_prob: f64, - /// Probability of a write failure on pending decision upsert. - pub decision_write_failure_prob: f64, - /// Probability of a read failure on pending decision load. - pub decision_read_failure_prob: f64, - /// Probability of a failure when deleting a spec (cleanup path). - pub cleanup_failure_prob: f64, - /// Probability of a read failure on WASM module load. - pub wasm_read_failure_prob: f64, - } - - impl SimPlatformFaultConfig { - /// No fault injection — all operations succeed. - pub fn none() -> Self { - Self { - spec_write_failure_prob: 0.0, - spec_read_failure_prob: 0.0, - policy_write_failure_prob: 0.0, - policy_read_failure_prob: 0.0, - app_record_failure_prob: 0.0, - app_list_failure_prob: 0.0, - decision_write_failure_prob: 0.0, - decision_read_failure_prob: 0.0, - cleanup_failure_prob: 0.0, - wasm_read_failure_prob: 0.0, - } - } - - /// Heavy fault injection for stress testing. - pub fn heavy() -> Self { - Self { - spec_write_failure_prob: 0.05, - spec_read_failure_prob: 0.02, - policy_write_failure_prob: 0.05, - policy_read_failure_prob: 0.02, - app_record_failure_prob: 0.03, - app_list_failure_prob: 0.02, - decision_write_failure_prob: 0.04, - decision_read_failure_prob: 0.02, - cleanup_failure_prob: 0.03, - wasm_read_failure_prob: 0.02, - } - } - } - - impl Default for SimPlatformFaultConfig { - fn default() -> Self { - Self::none() - } - } - - /// In-memory, deterministic platform store for DST. - /// - /// Implements [`PlatformStore`] trait. All operations resolve immediately. - /// Fault injection controlled by [`DeterministicRng`]. - /// - /// Uses `BTreeMap`/`BTreeSet` exclusively (no `HashMap`/`HashSet`) for - /// deterministic iteration order. - #[derive(Clone)] - pub struct SimPlatformStore { - inner: Arc>, - } - - struct SimPlatformStoreInner { - /// Deterministic RNG for fault injection. - rng: DeterministicRng, - /// Fault injection configuration. - faults: SimPlatformFaultConfig, - /// Specs keyed by (tenant, entity_type). - specs: BTreeMap<(String, String), SpecRow>, - /// Verification cache: (tenant, entity_type) -> (content_hash, verified). - verification_cache: BTreeMap<(String, String), (String, bool)>, - /// Cedar policies keyed by tenant. - policies: BTreeMap, - /// Granular Cedar policy rows keyed by (tenant, policy_id). - policy_entries: BTreeMap<(String, String), PolicyEntryRow>, - /// Cross-invariant definitions keyed by tenant. - constraints: BTreeMap, - /// Installed apps: (tenant, app_name). - installed_apps: BTreeSet<(String, String)>, - /// Installed app digest metadata keyed by (tenant, app_name). - installed_app_records: BTreeMap<(String, String), InstalledAppRecord>, - /// Pending decisions: id -> JSON data. - pending_decisions: BTreeMap, - /// WASM modules keyed by (tenant, module_name). - wasm_modules: BTreeMap<(String, String), WasmModuleRow>, - } - - impl SimPlatformStore { - /// Create a new `SimPlatformStore` with the given seed and fault config. - pub fn new(seed: u64, faults: SimPlatformFaultConfig) -> Self { - Self { - inner: Arc::new(Mutex::new(SimPlatformStoreInner { - rng: DeterministicRng::new(seed), - faults, - specs: BTreeMap::new(), - verification_cache: BTreeMap::new(), - policies: BTreeMap::new(), - policy_entries: BTreeMap::new(), - constraints: BTreeMap::new(), - installed_apps: BTreeSet::new(), - installed_app_records: BTreeMap::new(), - pending_decisions: BTreeMap::new(), - wasm_modules: BTreeMap::new(), - })), - } - } - - /// Create a `SimPlatformStore` with no fault injection. - pub fn no_faults(seed: u64) -> Self { - Self::new(seed, SimPlatformFaultConfig::none()) - } - - /// Temporarily disable all fault injection. - /// - /// Returns the previous config so it can be restored. Useful for - /// invariant checks that must read the store reliably. - pub fn disable_faults(&self) -> SimPlatformFaultConfig { - let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock - let prev = inner.faults.clone(); - inner.faults = SimPlatformFaultConfig::none(); - prev - } - - /// Restore a previously saved fault config. - pub fn restore_faults(&self, faults: SimPlatformFaultConfig) { - let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock - inner.faults = faults; - } - - /// Seed a granular policy row for recovery tests. - pub fn insert_policy_entry_for_test( - &self, - tenant: &str, - policy_id: &str, - cedar_text: &str, - enabled: bool, - ) { - let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock - inner.policy_entries.insert( - (tenant.to_string(), policy_id.to_string()), - PolicyEntryRow { - tenant: tenant.to_string(), - policy_id: policy_id.to_string(), - cedar_text: cedar_text.to_string(), - enabled, - }, - ); - } - } - - impl std::fmt::Debug for SimPlatformStore { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock - f.debug_struct("SimPlatformStore") - .field("specs", &inner.specs.len()) - .field("policies", &inner.policies.len()) - .field("policy_entries", &inner.policy_entries.len()) - .field("installed_apps", &inner.installed_apps.len()) - .field("wasm_modules", &inner.wasm_modules.len()) - .finish() - } - } - - #[async_trait::async_trait] - impl PlatformStore for SimPlatformStore { - async fn upsert_spec( - &self, - tenant: &str, - entity_type: &str, - ioa_source: &str, - csdl_xml: &str, - content_hash: &str, - ) -> Result<(), String> { - let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock - - let prob = inner.faults.spec_write_failure_prob; - if inner.rng.chance(prob) { - return Err("SimPlatformStore: injected spec write failure".into()); - } - - let key = (tenant.to_string(), entity_type.to_string()); - inner.specs.insert( - key, - SpecRow { - tenant: tenant.to_string(), - entity_type: entity_type.to_string(), - ioa_source: ioa_source.to_string(), - csdl_xml: Some(csdl_xml.to_string()), - content_hash: content_hash.to_string(), - committed: false, - }, - ); - Ok(()) - } - - async fn load_specs(&self) -> Result, String> { - let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock - - let prob = inner.faults.spec_read_failure_prob; - if inner.rng.chance(prob) { - return Err("SimPlatformStore: injected spec read failure".into()); - } - - Ok(inner - .specs - .values() - .filter(|s| s.committed) - .cloned() - .collect()) - } - - async fn delete_spec(&self, tenant: &str, entity_type: &str) -> Result<(), String> { - let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock - let prob = inner.faults.cleanup_failure_prob; - if inner.rng.chance(prob) { - return Err("SimPlatformStore: injected cleanup failure".into()); - } - inner - .specs - .remove(&(tenant.to_string(), entity_type.to_string())); - Ok(()) - } - - async fn commit_specs(&self, tenant: &str) -> Result<(), String> { - let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock - for spec in inner.specs.values_mut() { - if spec.tenant == tenant { - spec.committed = true; - } - } - Ok(()) - } - - async fn delete_uncommitted_specs(&self) -> Result { - let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock - let before = inner.specs.len(); - inner.specs.retain(|_, s| s.committed); - Ok(before - inner.specs.len()) - } - - async fn load_verification_cache( - &self, - tenant: &str, - ) -> Result, String> { - let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock - - let prob = inner.faults.spec_read_failure_prob; - if inner.rng.chance(prob) { - return Err("SimPlatformStore: injected verification cache read failure".into()); - } - - let mut cache = BTreeMap::new(); - for ((t, et), (hash, verified)) in &inner.verification_cache { - if t == tenant { - cache.insert(et.clone(), (hash.clone(), *verified)); - } - } - Ok(cache) - } - - async fn persist_spec_verification( - &self, - tenant: &str, - entity_type: &str, - update: SpecVerificationUpdate<'_>, - ) -> Result<(), String> { - let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock - - let prob = inner.faults.spec_write_failure_prob; - if inner.rng.chance(prob) { - return Err("SimPlatformStore: injected verification write failure".into()); - } - - let key = (tenant.to_string(), entity_type.to_string()); - inner - .verification_cache - .insert(key, (update.status.to_string(), update.verified)); - Ok(()) - } - - async fn upsert_tenant_policy( - &self, - tenant: &str, - policy_text: &str, - ) -> Result<(), String> { - let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock - - let prob = inner.faults.policy_write_failure_prob; - if inner.rng.chance(prob) { - return Err("SimPlatformStore: injected policy write failure".into()); - } - - inner - .policies - .insert(tenant.to_string(), policy_text.to_string()); - Ok(()) - } - - async fn upsert_tenant_constraints( - &self, - tenant: &str, - cross_invariants_toml: &str, - ) -> Result<(), String> { - let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock - - let prob = inner.faults.policy_write_failure_prob; - if inner.rng.chance(prob) { - return Err("SimPlatformStore: injected constraints write failure".into()); - } - - inner - .constraints - .insert(tenant.to_string(), cross_invariants_toml.to_string()); - Ok(()) - } - - async fn load_tenant_policies(&self) -> Result, String> { - let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock - - let prob = inner.faults.policy_read_failure_prob; - if inner.rng.chance(prob) { - return Err("SimPlatformStore: injected policy read failure".into()); - } - - Ok(inner - .policies - .iter() - .map(|(k, v)| (k.clone(), v.clone())) - .collect()) - } - - async fn load_policy_entries(&self) -> Result, String> { - let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock - - let prob = inner.faults.policy_read_failure_prob; - if inner.rng.chance(prob) { - return Err("SimPlatformStore: injected granular policy read failure".into()); - } - - Ok(inner.policy_entries.values().cloned().collect()) - } - - async fn is_app_installed(&self, tenant: &str, app_name: &str) -> Result { - let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock - - let prob = inner.faults.app_list_failure_prob; - if inner.rng.chance(prob) { - return Err("SimPlatformStore: injected app query failure".into()); - } - - Ok(inner - .installed_apps - .contains(&(tenant.to_string(), app_name.to_string()))) - } - - async fn record_installed_app(&self, tenant: &str, app_name: &str) -> Result<(), String> { - let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock - - let prob = inner.faults.app_record_failure_prob; - if inner.rng.chance(prob) { - return Err("SimPlatformStore: injected app record failure".into()); - } - - inner - .installed_apps - .insert((tenant.to_string(), app_name.to_string())); - Ok(()) - } - - async fn record_installed_app_metadata( - &self, - record: &InstalledAppRecord, - ) -> Result<(), String> { - let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock - - let prob = inner.faults.app_record_failure_prob; - if inner.rng.chance(prob) { - return Err("SimPlatformStore: injected app metadata record failure".into()); - } - - let key = (record.tenant.clone(), record.app_name.clone()); - inner.installed_apps.insert(key.clone()); - inner.installed_app_records.insert(key, record.clone()); - Ok(()) - } - - async fn get_installed_app( - &self, - tenant: &str, - app_name: &str, - ) -> Result, String> { - let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock - - let prob = inner.faults.app_list_failure_prob; - if inner.rng.chance(prob) { - return Err("SimPlatformStore: injected app metadata read failure".into()); - } - - Ok(inner - .installed_app_records - .get(&(tenant.to_string(), app_name.to_string())) - .cloned()) - } - - async fn list_all_installed_apps(&self) -> Result, String> { - let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock - - let prob = inner.faults.app_list_failure_prob; - if inner.rng.chance(prob) { - return Err("SimPlatformStore: injected app list failure".into()); - } - - Ok(inner.installed_apps.iter().cloned().collect()) - } - - async fn upsert_pending_decision( - &self, - id: &str, - tenant: &str, - status: &str, - data: &str, - ) -> Result<(), String> { - let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock - - let prob = inner.faults.decision_write_failure_prob; - if inner.rng.chance(prob) { - return Err("SimPlatformStore: injected decision write failure".into()); - } - - inner.pending_decisions.insert( - id.to_string(), - (tenant.to_string(), status.to_string(), data.to_string()), - ); - Ok(()) - } - - async fn load_pending_decisions(&self, limit: usize) -> Result, String> { - let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock - - let prob = inner.faults.decision_read_failure_prob; - if inner.rng.chance(prob) { - return Err("SimPlatformStore: injected decision read failure".into()); - } - - Ok(inner - .pending_decisions - .values() - .rev() - .take(limit) - .map(|(_, _, data)| data.clone()) - .collect()) - } - - async fn load_all_wasm_modules(&self, tenant: &str) -> Result, String> { - let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock - - let prob = inner.faults.wasm_read_failure_prob; - if inner.rng.chance(prob) { - return Err("SimPlatformStore: injected WASM read failure".into()); - } - - Ok(inner - .wasm_modules - .values() - .filter(|m| m.tenant == tenant) - .cloned() - .collect()) - } - - async fn load_wasm_modules_all_tenants(&self) -> Result, String> { - let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock - - let prob = inner.faults.wasm_read_failure_prob; - if inner.rng.chance(prob) { - return Err("SimPlatformStore: injected WASM read failure".into()); - } - - Ok(inner.wasm_modules.values().cloned().collect()) - } - - async fn upsert_wasm_module( - &self, - tenant: &str, - name: &str, - bytes: &[u8], - hash: &str, - source: &str, - ) -> Result<(), String> { - let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock - - let prob = inner.faults.spec_write_failure_prob; - if inner.rng.chance(prob) { - return Err("SimPlatformStore: injected WASM write failure".into()); - } - - let key = (tenant.to_string(), name.to_string()); - inner.wasm_modules.insert( - key, - WasmModuleRow { - tenant: tenant.to_string(), - module_name: name.to_string(), - wasm_bytes: bytes.to_vec(), - sha256_hash: hash.to_string(), - source: source.to_string(), - }, - ); - Ok(()) - } - } -} diff --git a/crates/temper-server/src/platform_store/sim/mod.rs b/crates/temper-server/src/platform_store/sim/mod.rs new file mode 100644 index 000000000..c39d46e4c --- /dev/null +++ b/crates/temper-server/src/platform_store/sim/mod.rs @@ -0,0 +1,417 @@ +//! Deterministic in-memory platform store for simulation. + +mod platform_store_impl; + +#[cfg(test)] +mod tests; + +use super::*; +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::{Arc, Mutex}; +use temper_runtime::scheduler::sim_now; +use temper_store_sim::DeterministicRng; + +/// Fault injection configuration for platform store simulation. +/// +/// Controls the probability of injected failures during platform store +/// operations. All probabilities are in \[0.0, 1.0\]. +#[derive(Debug, Clone)] +pub struct SimPlatformFaultConfig { + /// Probability of a write failure on spec upsert. + pub spec_write_failure_prob: f64, + /// Probability of a read failure on spec load. + pub spec_read_failure_prob: f64, + /// Probability of a write failure on policy upsert. + pub policy_write_failure_prob: f64, + /// Probability of a read failure on policy load. + pub policy_read_failure_prob: f64, + /// Probability of a failure recording an installed app. + pub app_record_failure_prob: f64, + /// Probability of a failure listing installed apps. + pub app_list_failure_prob: f64, + /// Probability of a write failure on pending decision upsert. + pub decision_write_failure_prob: f64, + /// Probability of a read failure on pending decision load. + pub decision_read_failure_prob: f64, + /// Probability of a failure when deleting a spec (cleanup path). + pub cleanup_failure_prob: f64, + /// Probability of a read failure on WASM module load. + pub wasm_read_failure_prob: f64, +} + +impl SimPlatformFaultConfig { + /// No fault injection — all operations succeed. + pub fn none() -> Self { + Self { + spec_write_failure_prob: 0.0, + spec_read_failure_prob: 0.0, + policy_write_failure_prob: 0.0, + policy_read_failure_prob: 0.0, + app_record_failure_prob: 0.0, + app_list_failure_prob: 0.0, + decision_write_failure_prob: 0.0, + decision_read_failure_prob: 0.0, + cleanup_failure_prob: 0.0, + wasm_read_failure_prob: 0.0, + } + } + + /// Heavy fault injection for stress testing. + pub fn heavy() -> Self { + Self { + spec_write_failure_prob: 0.05, + spec_read_failure_prob: 0.02, + policy_write_failure_prob: 0.05, + policy_read_failure_prob: 0.02, + app_record_failure_prob: 0.03, + app_list_failure_prob: 0.02, + decision_write_failure_prob: 0.04, + decision_read_failure_prob: 0.02, + cleanup_failure_prob: 0.03, + wasm_read_failure_prob: 0.02, + } + } +} + +impl Default for SimPlatformFaultConfig { + fn default() -> Self { + Self::none() + } +} + +/// In-memory, deterministic platform store for DST. +/// +/// Implements [`PlatformStore`] trait. All operations resolve immediately. +/// Fault injection controlled by [`DeterministicRng`]. +/// +/// Uses `BTreeMap`/`BTreeSet` exclusively (no `HashMap`/`HashSet`) for +/// deterministic iteration order. +#[derive(Clone)] +pub struct SimPlatformStore { + inner: Arc>, +} + +struct SimPlatformStoreInner { + /// Deterministic RNG for fault injection. + rng: DeterministicRng, + /// Fault injection configuration. + faults: SimPlatformFaultConfig, + /// Number of upcoming atomic spec publications that fail before any + /// durable mutation. + pending_spec_publication_failures: usize, + /// Number of upcoming atomic publications that durably commit but + /// return an outcome-ambiguous error to the caller. + pending_spec_publication_postcommit_failures: usize, + /// Specs keyed by (tenant, entity_type). + specs: BTreeMap<(String, String), SpecRow>, + /// Verification cache: (tenant, entity_type) -> (content_hash, verified). + verification_cache: BTreeMap<(String, String), (String, bool)>, + /// Cedar policies keyed by tenant. + policies: BTreeMap, + /// Granular Cedar policy rows keyed by (tenant, policy_id). + policy_entries: BTreeMap<(String, String), PolicyEntryRow>, + /// Cross-invariant definitions keyed by tenant. + constraints: BTreeMap, + /// Installed apps: (tenant, app_name). + installed_apps: BTreeSet<(String, String)>, + /// Installed app digest metadata keyed by (tenant, app_name). + installed_app_records: BTreeMap<(String, String), InstalledAppRecord>, + /// Pending decisions: id -> JSON data. + pending_decisions: BTreeMap, + /// WASM modules keyed by (tenant, module_name). + wasm_modules: BTreeMap<(String, String), WasmModuleRow>, +} + +impl SimPlatformStore { + /// Create a new `SimPlatformStore` with the given seed and fault config. + pub fn new(seed: u64, faults: SimPlatformFaultConfig) -> Self { + Self { + inner: Arc::new(Mutex::new(SimPlatformStoreInner { + rng: DeterministicRng::new(seed), + faults, + pending_spec_publication_failures: 0, + pending_spec_publication_postcommit_failures: 0, + specs: BTreeMap::new(), + verification_cache: BTreeMap::new(), + policies: BTreeMap::new(), + policy_entries: BTreeMap::new(), + constraints: BTreeMap::new(), + installed_apps: BTreeSet::new(), + installed_app_records: BTreeMap::new(), + pending_decisions: BTreeMap::new(), + wasm_modules: BTreeMap::new(), + })), + } + } + + /// Create a `SimPlatformStore` with no fault injection. + pub fn no_faults(seed: u64) -> Self { + Self::new(seed, SimPlatformFaultConfig::none()) + } + + /// Temporarily disable all fault injection. + /// + /// Returns the previous config so it can be restored. Useful for + /// invariant checks that must read the store reliably. + pub fn disable_faults(&self) -> SimPlatformFaultConfig { + let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock + let prev = inner.faults.clone(); + inner.faults = SimPlatformFaultConfig::none(); + prev + } + + /// Restore a previously saved fault config. + pub fn restore_faults(&self, faults: SimPlatformFaultConfig) { + let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock + inner.faults = faults; + } + + /// Fail the next `count` atomic spec publications before mutation. + /// `count == 0` clears the deterministic fault. + pub fn fail_next_spec_publications(&self, count: usize) { + self.inner + .lock() + .expect("SimPlatformStore lock poisoned") // ci-ok: infallible lock + .pending_spec_publication_failures = count; + } + + /// Commit the next `count` atomic spec publications, then return an + /// error as if the commit acknowledgement were lost. `count == 0` + /// clears the deterministic fault. + pub fn fail_next_spec_publications_after_commit(&self, count: usize) { + self.inner + .lock() + .expect("SimPlatformStore lock poisoned") // ci-ok: infallible lock + .pending_spec_publication_postcommit_failures = count; + } + + /// Seed a granular policy row for recovery tests. + pub fn insert_policy_entry_for_test( + &self, + tenant: &str, + policy_id: &str, + cedar_text: &str, + enabled: bool, + ) { + let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock + inner.policy_entries.insert( + (tenant.to_string(), policy_id.to_string()), + PolicyEntryRow { + tenant: tenant.to_string(), + policy_id: policy_id.to_string(), + cedar_text: cedar_text.to_string(), + created_by: "test".to_string(), + enabled, + }, + ); + } +} + +impl std::fmt::Debug for SimPlatformStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock + f.debug_struct("SimPlatformStore") + .field("specs", &inner.specs.len()) + .field("policies", &inner.policies.len()) + .field("policy_entries", &inner.policy_entries.len()) + .field("installed_apps", &inner.installed_apps.len()) + .field("wasm_modules", &inner.wasm_modules.len()) + .finish() + } +} + +impl SimPlatformStore { + #[expect( + clippy::too_many_arguments, + reason = "simulation mirrors the atomic platform publication boundary" + )] + async fn publish_specs_inner( + &self, + tenant: &str, + specs: &[SpecPublication<'_>], + mode: SpecPublicationMode, + constraints: TenantConstraintsPublication<'_>, + policy: TenantPolicyPublication<'_>, + os_app: Option>, + policy_generation: Option>, + wasm_modules: &[WasmPublication<'_>], + ) -> Result, String> { + let mut seen_types = BTreeSet::new(); + for spec in specs { + if !seen_types.insert(spec.entity_type) { + return Err(format!( + "duplicate entity type {} in spec publication for tenant {tenant}", + spec.entity_type + )); + } + } + let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock + if inner.pending_spec_publication_failures > 0 { + inner.pending_spec_publication_failures -= 1; + return Err("SimPlatformStore: injected spec publication failure".into()); + } + let prob = inner.faults.spec_write_failure_prob; + if inner.rng.chance(prob) { + return Err("SimPlatformStore: injected spec publication failure".into()); + } + if matches!(policy, TenantPolicyPublication::Replace(_)) { + let policy_prob = inner.faults.policy_write_failure_prob; + if inner.rng.chance(policy_prob) { + return Err("SimPlatformStore: injected policy publication failure".into()); + } + } + if os_app.is_some() { + let app_prob = inner.faults.app_record_failure_prob; + if inner.rng.chance(app_prob) { + return Err("SimPlatformStore: injected app publication failure".into()); + } + } + let mut seen_modules = BTreeSet::new(); + for module in wasm_modules { + if !seen_modules.insert(module.module_name) { + return Err(format!( + "duplicate WASM module {} in spec publication for tenant {tenant}", + module.module_name + )); + } + } + let granular_policy = policy_generation + .map(|publication| (publication.policy_owner, publication.policy_entries)) + .or_else(|| { + os_app.and_then(|publication| { + publication + .policy_owner + .map(|owner| (owner, publication.policy_entries)) + }) + }); + if let Some((_, entries)) = granular_policy { + let mut seen_policy_ids = BTreeSet::new(); + for entry in entries { + if !seen_policy_ids.insert(entry.policy_id) { + return Err(format!( + "duplicate Cedar policy {} in spec publication for tenant {tenant}", + entry.policy_id + )); + } + } + } + let fail_after_commit = if inner.pending_spec_publication_postcommit_failures > 0 { + inner.pending_spec_publication_postcommit_failures -= 1; + true + } else { + false + }; + let incoming_types = specs + .iter() + .map(|spec| spec.entity_type) + .collect::>(); + let removed_entity_types = if mode == SpecPublicationMode::Replace { + inner + .specs + .keys() + .filter(|(row_tenant, entity_type)| { + row_tenant == tenant && !incoming_types.contains(entity_type.as_str()) + }) + .map(|(_, entity_type)| entity_type.clone()) + .collect::>() + } else { + Vec::new() + }; + for spec in specs { + inner.specs.insert( + (tenant.to_string(), spec.entity_type.to_string()), + SpecRow { + tenant: tenant.to_string(), + entity_type: spec.entity_type.to_string(), + ioa_source: spec.ioa_source.to_string(), + csdl_xml: Some(spec.csdl_xml.to_string()), + content_hash: spec.content_hash.to_string(), + committed: true, + }, + ); + } + for entity_type in &removed_entity_types { + inner + .specs + .remove(&(tenant.to_string(), entity_type.clone())); + } + match constraints { + TenantConstraintsPublication::Preserve => {} + TenantConstraintsPublication::Replace(Some(source)) => { + inner + .constraints + .insert(tenant.to_string(), source.to_string()); + } + TenantConstraintsPublication::Replace(None) => { + inner.constraints.remove(tenant); + } + } + if let TenantPolicyPublication::Replace(policy) = policy { + inner + .policies + .insert(tenant.to_string(), policy.to_string()); + } + if let Some((owner, entries)) = granular_policy { + inner + .policy_entries + .retain(|(row_tenant, _), row| row_tenant != tenant || row.created_by != owner); + for entry in entries { + inner.policy_entries.insert( + (tenant.to_string(), entry.policy_id.to_string()), + PolicyEntryRow { + tenant: tenant.to_string(), + policy_id: entry.policy_id.to_string(), + cedar_text: entry.cedar_text.to_string(), + created_by: entry.created_by.to_string(), + enabled: true, + }, + ); + } + } + if let Some(publication) = os_app { + let key = (tenant.to_string(), publication.record.app_name.clone()); + let now = sim_now().to_rfc3339(); + let mut record = publication.record.clone(); + record.installed_at = inner + .installed_app_records + .get(&key) + .and_then(|existing| existing.installed_at.clone()) + .or_else(|| Some(now.clone())); + record.last_reconciled_at = Some(now); + inner.installed_apps.insert(key.clone()); + inner.installed_app_records.insert(key, record); + } + for module in wasm_modules { + let key = (tenant.to_string(), module.module_name.to_string()); + let replace_uploaded_wasm = module.source == "bundled-replace-upload"; + let persisted_source = if replace_uploaded_wasm { + "bundled" + } else { + module.source + }; + let should_replace = inner.wasm_modules.get(&key).is_none_or(|existing| { + existing.sha256_hash != module.sha256_hash + && (replace_uploaded_wasm + || persisted_source == "upload" + || existing.source == "bundled") + }); + if should_replace { + inner.wasm_modules.insert( + key, + WasmModuleRow { + tenant: tenant.to_string(), + module_name: module.module_name.to_string(), + wasm_bytes: module.wasm_bytes.to_vec(), + sha256_hash: module.sha256_hash.to_string(), + source: persisted_source.to_string(), + }, + ); + } + } + if fail_after_commit { + Err("SimPlatformStore: injected post-commit publication failure".into()) + } else { + Ok(removed_entity_types) + } + } +} diff --git a/crates/temper-server/src/platform_store/sim/platform_store_impl.rs b/crates/temper-server/src/platform_store/sim/platform_store_impl.rs new file mode 100644 index 000000000..ba57a516e --- /dev/null +++ b/crates/temper-server/src/platform_store/sim/platform_store_impl.rs @@ -0,0 +1,383 @@ +//! PlatformStore implementation for the deterministic simulator. + +use super::*; + +#[async_trait::async_trait] +impl PlatformStore for SimPlatformStore { + async fn upsert_spec( + &self, + tenant: &str, + entity_type: &str, + ioa_source: &str, + csdl_xml: &str, + content_hash: &str, + ) -> Result<(), String> { + let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock + + let prob = inner.faults.spec_write_failure_prob; + if inner.rng.chance(prob) { + return Err("SimPlatformStore: injected spec write failure".into()); + } + + let key = (tenant.to_string(), entity_type.to_string()); + inner.specs.insert( + key, + SpecRow { + tenant: tenant.to_string(), + entity_type: entity_type.to_string(), + ioa_source: ioa_source.to_string(), + csdl_xml: Some(csdl_xml.to_string()), + content_hash: content_hash.to_string(), + committed: false, + }, + ); + Ok(()) + } + + async fn publish_specs( + &self, + tenant: &str, + specs: &[SpecPublication<'_>], + mode: SpecPublicationMode, + constraints: TenantConstraintsPublication<'_>, + policy: TenantPolicyPublication<'_>, + os_app: Option>, + policy_generation: Option>, + wasm_modules: &[WasmPublication<'_>], + ) -> Result, String> { + self.publish_specs_inner( + tenant, + specs, + mode, + constraints, + policy, + os_app, + policy_generation, + wasm_modules, + ) + .await + } + async fn load_specs(&self) -> Result, String> { + let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock + + let prob = inner.faults.spec_read_failure_prob; + if inner.rng.chance(prob) { + return Err("SimPlatformStore: injected spec read failure".into()); + } + + Ok(inner + .specs + .values() + .filter(|s| s.committed) + .cloned() + .collect()) + } + + async fn delete_spec(&self, tenant: &str, entity_type: &str) -> Result<(), String> { + let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock + let prob = inner.faults.cleanup_failure_prob; + if inner.rng.chance(prob) { + return Err("SimPlatformStore: injected cleanup failure".into()); + } + inner + .specs + .remove(&(tenant.to_string(), entity_type.to_string())); + Ok(()) + } + + async fn commit_specs(&self, tenant: &str) -> Result<(), String> { + let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock + for spec in inner.specs.values_mut() { + if spec.tenant == tenant { + spec.committed = true; + } + } + Ok(()) + } + + async fn delete_uncommitted_specs(&self) -> Result { + let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock + let before = inner.specs.len(); + inner.specs.retain(|_, s| s.committed); + Ok(before - inner.specs.len()) + } + + async fn load_verification_cache( + &self, + tenant: &str, + ) -> Result, String> { + let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock + + let prob = inner.faults.spec_read_failure_prob; + if inner.rng.chance(prob) { + return Err("SimPlatformStore: injected verification cache read failure".into()); + } + + let mut cache = BTreeMap::new(); + for ((t, et), (hash, verified)) in &inner.verification_cache { + if t == tenant { + cache.insert(et.clone(), (hash.clone(), *verified)); + } + } + Ok(cache) + } + + async fn persist_spec_verification( + &self, + tenant: &str, + entity_type: &str, + update: SpecVerificationUpdate<'_>, + ) -> Result<(), String> { + let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock + + let prob = inner.faults.spec_write_failure_prob; + if inner.rng.chance(prob) { + return Err("SimPlatformStore: injected verification write failure".into()); + } + + let key = (tenant.to_string(), entity_type.to_string()); + inner + .verification_cache + .insert(key, (update.status.to_string(), update.verified)); + Ok(()) + } + + async fn upsert_tenant_policy(&self, tenant: &str, policy_text: &str) -> Result<(), String> { + let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock + + let prob = inner.faults.policy_write_failure_prob; + if inner.rng.chance(prob) { + return Err("SimPlatformStore: injected policy write failure".into()); + } + + inner + .policies + .insert(tenant.to_string(), policy_text.to_string()); + Ok(()) + } + + async fn upsert_tenant_constraints( + &self, + tenant: &str, + cross_invariants_toml: &str, + ) -> Result<(), String> { + let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock + + let prob = inner.faults.policy_write_failure_prob; + if inner.rng.chance(prob) { + return Err("SimPlatformStore: injected constraints write failure".into()); + } + + inner + .constraints + .insert(tenant.to_string(), cross_invariants_toml.to_string()); + Ok(()) + } + + async fn load_tenant_policies(&self) -> Result, String> { + let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock + + let prob = inner.faults.policy_read_failure_prob; + if inner.rng.chance(prob) { + return Err("SimPlatformStore: injected policy read failure".into()); + } + + Ok(inner + .policies + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect()) + } + + async fn load_policy_entries(&self) -> Result, String> { + let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock + + let prob = inner.faults.policy_read_failure_prob; + if inner.rng.chance(prob) { + return Err("SimPlatformStore: injected granular policy read failure".into()); + } + + Ok(inner.policy_entries.values().cloned().collect()) + } + + async fn is_app_installed(&self, tenant: &str, app_name: &str) -> Result { + let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock + + let prob = inner.faults.app_list_failure_prob; + if inner.rng.chance(prob) { + return Err("SimPlatformStore: injected app query failure".into()); + } + + Ok(inner + .installed_apps + .contains(&(tenant.to_string(), app_name.to_string()))) + } + + async fn record_installed_app(&self, tenant: &str, app_name: &str) -> Result<(), String> { + let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock + + let prob = inner.faults.app_record_failure_prob; + if inner.rng.chance(prob) { + return Err("SimPlatformStore: injected app record failure".into()); + } + + inner + .installed_apps + .insert((tenant.to_string(), app_name.to_string())); + Ok(()) + } + + async fn record_installed_app_metadata( + &self, + record: &InstalledAppRecord, + ) -> Result<(), String> { + let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock + + let prob = inner.faults.app_record_failure_prob; + if inner.rng.chance(prob) { + return Err("SimPlatformStore: injected app metadata record failure".into()); + } + + let key = (record.tenant.clone(), record.app_name.clone()); + inner.installed_apps.insert(key.clone()); + inner.installed_app_records.insert(key, record.clone()); + Ok(()) + } + + async fn get_installed_app( + &self, + tenant: &str, + app_name: &str, + ) -> Result, String> { + let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock + + let prob = inner.faults.app_list_failure_prob; + if inner.rng.chance(prob) { + return Err("SimPlatformStore: injected app metadata read failure".into()); + } + + Ok(inner + .installed_app_records + .get(&(tenant.to_string(), app_name.to_string())) + .cloned()) + } + + async fn list_all_installed_apps(&self) -> Result, String> { + let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock + + let prob = inner.faults.app_list_failure_prob; + if inner.rng.chance(prob) { + return Err("SimPlatformStore: injected app list failure".into()); + } + + Ok(inner.installed_apps.iter().cloned().collect()) + } + + async fn upsert_pending_decision( + &self, + id: &str, + tenant: &str, + status: &str, + data: &str, + ) -> Result<(), String> { + let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock + + let prob = inner.faults.decision_write_failure_prob; + if inner.rng.chance(prob) { + return Err("SimPlatformStore: injected decision write failure".into()); + } + + inner.pending_decisions.insert( + id.to_string(), + (tenant.to_string(), status.to_string(), data.to_string()), + ); + Ok(()) + } + + async fn load_pending_decisions(&self, limit: usize) -> Result, String> { + let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock + + let prob = inner.faults.decision_read_failure_prob; + if inner.rng.chance(prob) { + return Err("SimPlatformStore: injected decision read failure".into()); + } + + Ok(inner + .pending_decisions + .values() + .rev() + .take(limit) + .map(|(_, _, data)| data.clone()) + .collect()) + } + + async fn load_all_wasm_modules(&self, tenant: &str) -> Result, String> { + let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock + + let prob = inner.faults.wasm_read_failure_prob; + if inner.rng.chance(prob) { + return Err("SimPlatformStore: injected WASM read failure".into()); + } + + Ok(inner + .wasm_modules + .values() + .filter(|m| m.tenant == tenant) + .cloned() + .collect()) + } + + async fn load_wasm_modules_all_tenants(&self) -> Result, String> { + let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock + + let prob = inner.faults.wasm_read_failure_prob; + if inner.rng.chance(prob) { + return Err("SimPlatformStore: injected WASM read failure".into()); + } + + Ok(inner.wasm_modules.values().cloned().collect()) + } + + async fn upsert_wasm_module( + &self, + tenant: &str, + name: &str, + bytes: &[u8], + hash: &str, + source: &str, + ) -> Result<(), String> { + let mut inner = self.inner.lock().expect("SimPlatformStore lock poisoned"); // ci-ok: infallible lock + + let prob = inner.faults.spec_write_failure_prob; + if inner.rng.chance(prob) { + return Err("SimPlatformStore: injected WASM write failure".into()); + } + + let key = (tenant.to_string(), name.to_string()); + let replace_uploaded_wasm = source == "bundled-replace-upload"; + let persisted_source = if replace_uploaded_wasm { + "bundled" + } else { + source + }; + let should_replace = inner.wasm_modules.get(&key).is_none_or(|existing| { + existing.sha256_hash != hash + && (replace_uploaded_wasm + || persisted_source == "upload" + || existing.source == "bundled") + }); + if should_replace { + inner.wasm_modules.insert( + key, + WasmModuleRow { + tenant: tenant.to_string(), + module_name: name.to_string(), + wasm_bytes: bytes.to_vec(), + sha256_hash: hash.to_string(), + source: persisted_source.to_string(), + }, + ); + } + Ok(()) + } +} diff --git a/crates/temper-server/src/platform_store/sim/tests.rs b/crates/temper-server/src/platform_store/sim/tests.rs new file mode 100644 index 000000000..319e83eb5 --- /dev/null +++ b/crates/temper-server/src/platform_store/sim/tests.rs @@ -0,0 +1,102 @@ +//! Simulated atomic-publication regression tests. + +use super::*; + +#[tokio::test] +async fn atomic_publication_commits_specs_and_wasm_before_ambiguous_error() { + let store = SimPlatformStore::no_faults(238); + store.fail_next_spec_publications_after_commit(1); + let spec = SpecPublication { + entity_type: "Widget", + ioa_source: "[automaton]\nname = \"Widget\"", + csdl_xml: "", + content_hash: "spec-hash", + }; + let wasm = WasmPublication { + module_name: "widget_hook", + wasm_bytes: b"\0asm\x01\0\0\0", + sha256_hash: "wasm-hash", + source: "bundled", + }; + + let error = store + .publish_specs( + "default", + &[spec], + SpecPublicationMode::Merge, + TenantConstraintsPublication::Preserve, + TenantPolicyPublication::Preserve, + None, + None, + &[wasm], + ) + .await + .expect_err("post-commit acknowledgement must be ambiguous"); + assert!(error.contains("post-commit")); + assert_eq!(store.load_specs().await.expect("load specs").len(), 1); + let modules = store + .load_all_wasm_modules("default") + .await + .expect("load WASM generation"); + assert_eq!(modules.len(), 1); + assert_eq!(modules[0].sha256_hash, "wasm-hash"); + assert_eq!(modules[0].source, "bundled"); +} + +#[tokio::test] +async fn atomic_publication_matches_uploaded_wasm_source_precedence() { + let store = SimPlatformStore::no_faults(239); + store + .upsert_wasm_module("default", "hook", b"upload", "upload-hash", "upload") + .await + .expect("seed uploaded WASM"); + let bundled = WasmPublication { + module_name: "hook", + wasm_bytes: b"bundled", + sha256_hash: "bundled-hash", + source: "bundled", + }; + store + .publish_specs( + "default", + &[], + SpecPublicationMode::Merge, + TenantConstraintsPublication::Preserve, + TenantPolicyPublication::Preserve, + None, + None, + &[bundled], + ) + .await + .expect("plain bundle publication"); + let preserved = store + .load_all_wasm_modules("default") + .await + .expect("load preserved upload"); + assert_eq!(preserved[0].sha256_hash, "upload-hash"); + assert_eq!(preserved[0].source, "upload"); + + let replacement = WasmPublication { + source: "bundled-replace-upload", + ..bundled + }; + store + .publish_specs( + "default", + &[], + SpecPublicationMode::Merge, + TenantConstraintsPublication::Preserve, + TenantPolicyPublication::Preserve, + None, + None, + &[replacement], + ) + .await + .expect("explicit bundle replacement"); + let replaced = store + .load_all_wasm_modules("default") + .await + .expect("load replaced bundle"); + assert_eq!(replaced[0].sha256_hash, "bundled-hash"); + assert_eq!(replaced[0].source, "bundled"); +} diff --git a/crates/temper-server/src/registry/mod.rs b/crates/temper-server/src/registry/mod.rs index 8319e7208..5dc8ad768 100644 --- a/crates/temper-server/src/registry/mod.rs +++ b/crates/temper-server/src/registry/mod.rs @@ -138,7 +138,10 @@ impl SpecRegistry { /// submission are preserved. This is the correct mode for /// `load-inline` (agent `submit_specs`), where the agent only submits /// its own entities and should not wipe platform types. - #[allow(clippy::too_many_arguments)] + #[expect( + clippy::too_many_arguments, + reason = "atomic publication must receive the complete registry generation payload" + )] #[instrument(skip_all, fields(otel.name = "registry.try_register_tenant_with_reactions_and_constraints"))] pub fn try_register_tenant_with_reactions_and_constraints( &mut self, @@ -149,6 +152,34 @@ impl SpecRegistry { reactions: Vec, cross_invariants_source: Option, merge: bool, + ) -> Result<(), RegistryError> { + self.try_register_tenant_with_reactions_constraints_and_key_epochs( + tenant, + csdl, + csdl_xml, + ioa_sources, + reactions, + cross_invariants_source, + merge, + &BTreeMap::new(), + ) + } + + /// Register specs whose declared-key contracts were already fenced in + /// durable storage, attaching each monotonic activation epoch to the table + /// before it becomes visible to actors. + #[allow(clippy::too_many_arguments)] + #[instrument(skip_all, fields(otel.name = "registry.try_register_tenant_with_key_epochs"))] + pub fn try_register_tenant_with_reactions_constraints_and_key_epochs( + &mut self, + tenant: impl Into, + csdl: CsdlDocument, + csdl_xml: String, + ioa_sources: &[(&str, &str)], + reactions: Vec, + cross_invariants_source: Option, + merge: bool, + key_contract_activation_epochs: &BTreeMap, ) -> Result<(), RegistryError> { let tenant = tenant.into(); let tenant_name = tenant.to_string(); @@ -179,7 +210,69 @@ impl SpecRegistry { } } + // Parse and compile the full submission before mutating live tenant + // metadata. This prevents partial registration and leaves no fallible + // source work after durable key-contract activation. + let mut compiled_specs = Vec::with_capacity(ioa_sources.len()); + for (entity_type, ioa_source) in ioa_sources { + let automaton = + automaton::parse_automaton(ioa_source).map_err(|e| RegistryError::IoaParse { + tenant: tenant_name.clone(), + entity_type: (*entity_type).to_string(), + source: e.to_string(), + })?; + let mut table = TransitionTable::from_automaton(&automaton); + table.key_contract_activation_epoch = key_contract_activation_epochs + .get(*entity_type) + .copied() + .unwrap_or(0); + let integrations = automaton.integrations.clone(); + compiled_specs.push(( + (*entity_type).to_string(), + (*ioa_source).to_string(), + automaton, + integrations, + table, + )); + } + if let Some(existing_config) = self.tenants.get_mut(&tenant) { + let tenant_has_activated_contracts = existing_config + .entities + .values() + .any(|spec| spec.table().key_contract_activation_epoch > 0); + for (entity_type, _, _, _, table) in &compiled_specs { + if key_contract_activation_epochs.contains_key(entity_type) { + continue; + } + let attempted_signature = crate::key_index::declared_key_set_signature(&table.keys); + match existing_config.entities.get(entity_type) { + Some(existing) + if existing.table().key_contract_activation_epoch > 0 + && crate::key_index::declared_key_set_signature( + &existing.table().keys, + ) != attempted_signature => + { + return Err(RegistryError::KeyContractActivationRequired { + tenant: tenant_name.clone(), + entity_type: entity_type.clone(), + current_signature: Some(crate::key_index::declared_key_set_signature( + &existing.table().keys, + )), + attempted_signature, + }); + } + None if tenant_has_activated_contracts => { + return Err(RegistryError::KeyContractActivationRequired { + tenant: tenant_name.clone(), + entity_type: entity_type.clone(), + current_signature: None, + attempted_signature, + }); + } + _ => {} + } + } // Hot-reload path: swap tables on existing entities, add new ones. if merge { // Merge mode: combine incoming CSDL/entity-set-map with existing. @@ -211,18 +304,12 @@ impl SpecRegistry { existing_config.cross_invariants_source = cross_invariants_source; } - for (entity_type, ioa_source) in ioa_sources { - let automaton = automaton::parse_automaton(ioa_source).map_err(|e| { - RegistryError::IoaParse { - tenant: tenant_name.clone(), - entity_type: (*entity_type).to_string(), - source: e.to_string(), + for (entity_type, ioa_source, automaton, integrations, mut table) in compiled_specs { + if let Some(existing_spec) = existing_config.entities.get_mut(&entity_type) { + if !key_contract_activation_epochs.contains_key(&entity_type) { + table.key_contract_activation_epoch = + existing_spec.table().key_contract_activation_epoch; } - })?; - let table = TransitionTable::from_automaton(&automaton); - let integrations = automaton.integrations.clone(); - - if let Some(existing_spec) = existing_config.entities.get_mut(*entity_type) { // Hot-swap: write new table into the SAME RwLock that actors hold. let result = existing_spec.swap_controller().swap(table); tracing::info!( @@ -233,16 +320,16 @@ impl SpecRegistry { // Update metadata on the existing spec. existing_spec.automaton = automaton; existing_spec.integrations = integrations; - existing_spec.ioa_source = ioa_source.to_string(); + existing_spec.ioa_source = ioa_source; } else { // New entity type — create fresh EntitySpec. existing_config.entities.insert( - entity_type.to_string(), + entity_type, EntitySpec { automaton, integrations, swap: Arc::new(SwapController::new(table)), - ioa_source: ioa_source.to_string(), + ioa_source, }, ); } @@ -278,23 +365,14 @@ impl SpecRegistry { } else { // First registration: create new TenantConfig. let mut entities = BTreeMap::new(); - for (entity_type, ioa_source) in ioa_sources { - let automaton = automaton::parse_automaton(ioa_source).map_err(|e| { - RegistryError::IoaParse { - tenant: tenant_name.clone(), - entity_type: (*entity_type).to_string(), - source: e.to_string(), - } - })?; - let table = TransitionTable::from_automaton(&automaton); - let integrations = automaton.integrations.clone(); + for (entity_type, ioa_source, automaton, integrations, table) in compiled_specs { entities.insert( - entity_type.to_string(), + entity_type, EntitySpec { automaton, integrations, swap: Arc::new(SwapController::new(table)), - ioa_source: ioa_source.to_string(), + ioa_source, }, ); } diff --git a/crates/temper-server/src/registry/types.rs b/crates/temper-server/src/registry/types.rs index ad46887db..41a8e9db7 100644 --- a/crates/temper-server/src/registry/types.rs +++ b/crates/temper-server/src/registry/types.rs @@ -87,6 +87,14 @@ pub enum RegistryError { entity_type: String, source: String, }, + /// A hot swap changed a declared-key contract without first activating a + /// store epoch for the replacement table. + KeyContractActivationRequired { + tenant: String, + entity_type: String, + current_signature: Option, + attempted_signature: String, + }, } impl std::fmt::Display for RegistryError { @@ -108,6 +116,15 @@ impl std::fmt::Display for RegistryError { "failed to parse IOA for tenant '{tenant}', entity '{entity_type}': {source}" ) } + Self::KeyContractActivationRequired { + tenant, + entity_type, + current_signature, + attempted_signature, + } => write!( + f, + "key contract activation required for tenant '{tenant}', entity '{entity_type}' before changing {current_signature:?} to '{attempted_signature}'" + ), } } } diff --git a/crates/temper-server/src/request_context.rs b/crates/temper-server/src/request_context.rs index 88ed2d5cd..3e2db0e15 100644 --- a/crates/temper-server/src/request_context.rs +++ b/crates/temper-server/src/request_context.rs @@ -70,6 +70,9 @@ pub struct AgentContext { /// `workflow.run_id`, `producer.work_item_id`, or `support.ticket_id`. /// Temper core treats these keys as opaque correlation metadata. pub observation_metadata: BTreeMap, + /// Stable runtime generation inherited by immediate dispatch work. + /// Delayed timers detach this lease and retain only its generation token. + pub(crate) tenant_generation_lease: Option, } impl AgentContext { @@ -92,6 +95,7 @@ impl AgentContext { workflow_run_id: None, idempotency_key: None, observation_metadata: BTreeMap::new(), + tenant_generation_lease: None, } } @@ -127,6 +131,7 @@ impl AgentContext { workflow_run_id: None, idempotency_key: None, observation_metadata: BTreeMap::new(), + tenant_generation_lease: None, } } @@ -157,6 +162,7 @@ impl AgentContext { self.workflow_root_entity_id = parent.workflow_root_entity_id.clone(); self.workflow_run_id = parent.workflow_run_id.clone(); self.observation_metadata = parent.observation_metadata.clone(); + self.tenant_generation_lease = parent.tenant_generation_lease.clone(); self } @@ -196,6 +202,36 @@ impl AgentContext { } serde_json::to_string(&self.observation_metadata).ok() } + + pub(crate) fn with_tenant_generation_lease( + mut self, + lease: crate::state::TenantGenerationLease, + ) -> Self { + self.tenant_generation_lease = Some(lease); + self + } + + pub(crate) fn detach_tenant_generation_lease(&mut self) -> Option { + self.tenant_generation_lease + .take() + .map(|lease| lease.captured_generation()) + } + + /// Clone this context for immediate detached work while independently + /// retaining the originating request generation (or its explicit + /// publication-owned pending-generation token). + pub(crate) fn fork_tenant_generation_lease( + &self, + tenant: &temper_runtime::tenant::TenantId, + ) -> Option { + let lease = self + .tenant_generation_lease + .as_ref()? + .fork_immediate(tenant)?; + let mut forked = self.clone(); + forked.tenant_generation_lease = Some(lease); + Some(forked) + } } fn header_string(headers: &HeaderMap, name: &str) -> Option { @@ -256,6 +292,7 @@ pub(crate) fn extract_agent_context(headers: &HeaderMap) -> AgentContext { workflow_run_id, idempotency_key, observation_metadata: observation_metadata::extract(headers), + tenant_generation_lease: None, } } @@ -288,211 +325,5 @@ pub(crate) fn remote_parent_context(agent_ctx: &AgentContext) -> Option Router { .patch(odata::handle_odata_patch) .put(odata::handle_odata_put) .delete(odata::handle_odata_delete), - ); + ) + .route_layer(axum::middleware::from_fn_with_state( + state.clone(), + stable_tenant_generation, + )); let router = Router::new() .nest("/tdata", tdata) @@ -203,6 +211,17 @@ async fn http_endpoint_fallback( } } }; + let Some(generation_guard) = state.try_begin_tenant_request(&tenant_id).await else { + return publication_in_progress_response(); + }; + if state.spec_publication_gated(&tenant_id) { + return publication_in_progress_response(); + } + let generation_lease = crate::state::TenantGenerationLease::new( + &tenant_id, + generation_guard, + state.tenant_generation_version(&tenant_id), + ); let table = state.http_endpoint_tables.table_for(&tenant_id).await; let matched = table.match_request(method.as_str(), uri.path()).await; @@ -211,7 +230,17 @@ async fn http_endpoint_fallback( return http_404_response(uri.path()); }; - dispatch_matched_route(state, tenant_id, method, uri, headers, _body, route).await + dispatch_matched_route( + state, + tenant_id, + method, + uri, + headers, + _body, + route, + generation_lease, + ) + .await } /// Tenant for a header-less HttpEndpoint request: the registered @@ -239,6 +268,10 @@ fn http_endpoint_fallback_tenant<'a>(tenant_ids: &[&'a TenantId]) -> Option<&'a /// body that drains the guest's response-body handle, fire the /// WASM invocation, await the head, and stitch it into the /// response. +#[expect( + clippy::too_many_arguments, + reason = "matched HTTP route dispatch carries the captured request generation" +)] async fn dispatch_matched_route( state: ServerState, tenant_id: TenantId, @@ -247,6 +280,7 @@ async fn dispatch_matched_route( headers: HeaderMap, body: Body, route: crate::http_endpoint::MatchedRoute, + generation_lease: crate::state::TenantGenerationLease, ) -> Response { use temper_wasm::http_stream::HttpResponseHead; use temper_wasm::types::{HttpDispatchContext, WasmInvocationContext}; @@ -289,7 +323,9 @@ async fn dispatch_matched_route( // ADR-0057 inbound exchange end-to-end streaming — without it, // even SDK-streaming guests are bounded by the buffered limit. let pump_streams = streams.clone(); + let pump_generation = generation_lease.clone(); tokio::spawn(async move { + let _generation = pump_generation; use tokio_stream::StreamExt as _; let mut stream = body.into_data_stream(); while let Some(chunk_result) = stream.next().await { @@ -436,11 +472,14 @@ async fn dispatch_matched_route( route.route.requires_auth, action_bridge, result, + generation_lease, ) .await; } + let invoke_generation = generation_lease.clone(); let invoke_task = tokio::spawn(async move { + let _generation = invoke_generation; match engine .invoke_with_blobs( &invoke_hash, @@ -509,7 +548,9 @@ async fn dispatch_matched_route( // Build the axum response whose body drains the // kernel_response_body handle. let drain_streams = streams.clone(); + let response_generation = generation_lease; let body_stream = async_stream::stream! { + let _generation = response_generation; loop { match drain_streams.read(kernel_response_body).await { Ok(chunk) if chunk.is_empty() => break, @@ -527,6 +568,10 @@ async fn dispatch_matched_route( .expect("response builder") } +#[expect( + clippy::too_many_arguments, + reason = "action bridge result carries the captured request generation" +)] async fn dispatch_action_bridge_result( state: ServerState, tenant_id: TenantId, @@ -535,6 +580,7 @@ async fn dispatch_action_bridge_result( requires_auth: bool, bridge: crate::http_endpoint::HttpActionBridge, result: temper_wasm::types::WasmInvocationResult, + generation_lease: crate::state::TenantGenerationLease, ) -> Response { let callback_params = result.callback_params.clone(); let refs = bridge_git_receive_pack_refs(&callback_params); @@ -565,6 +611,7 @@ async fn dispatch_action_bridge_result( let action_params = bridge_action_params(&callback_params); let mut agent_ctx = crate::request_context::extract_agent_context(&headers); + agent_ctx = agent_ctx.with_tenant_generation_lease(generation_lease); let explicit_principal = headers.contains_key("x-temper-principal-kind") || headers.contains_key("x-temper-principal-id") || headers.contains_key("x-temper-principal-scopes"); diff --git a/crates/temper-server/src/router/tenant_generation.rs b/crates/temper-server/src/router/tenant_generation.rs new file mode 100644 index 000000000..3948ec795 --- /dev/null +++ b/crates/temper-server/src/router/tenant_generation.rs @@ -0,0 +1,114 @@ +//! Tenant-generation admission and publication barriers. + +use axum::extract::{Request, State}; +use axum::http::{Method, StatusCode}; +use axum::middleware::Next; +use axum::response::{IntoResponse, Response}; +use temper_runtime::tenant::TenantId; + +use crate::odata; +use crate::state::ServerState; + +/// Hold a tenant generation stable for the full OData handler. Publications +/// take the matching write side of this barrier, so an in-flight request sees +/// either the complete old generation or the complete new one—never a registry, +/// Cedar, reaction, and WASM mixture. +pub(super) async fn stable_tenant_generation( + State(state): State, + mut request: Request, + next: Next, +) -> Response { + let tenant = match odata::extract_tenant(request.headers(), &state) { + Ok(tenant) => tenant, + Err(error) => return error.into_response(), + }; + if state.spec_publication_gated(&tenant) { + if !gated_publication_retry_allowed(&state, &tenant, &request) { + return publication_in_progress_response(); + } + let Some(generation) = state.try_begin_tenant_request(&tenant).await else { + return publication_in_progress_response(); + }; + if !state.spec_publication_gated(&tenant) { + return publication_in_progress_response(); + } + let captured_generation = state.tenant_generation_version(&tenant); + let lease = + crate::state::TenantGenerationLease::new(&tenant, generation, captured_generation); + request.extensions_mut().insert(lease.clone()); + let response = next.run(request).await; + lease.release(); + return response; + } + let Some(generation) = state.try_begin_tenant_request(&tenant).await else { + return publication_in_progress_response(); + }; + if state.spec_publication_gated(&tenant) { + return publication_in_progress_response(); + } + let captured_generation = state.tenant_generation_version(&tenant); + let lease = crate::state::TenantGenerationLease::new(&tenant, generation, captured_generation); + request.extensions_mut().insert(lease.clone()); + let response = next.run(request).await; + lease.release(); + response +} + +fn gated_publication_retry_allowed( + state: &ServerState, + tenant: &TenantId, + request: &Request, +) -> bool { + if request.method() != Method::POST { + return false; + } + let Some(_idempotency_key) = request + .headers() + .get("idempotency-key") + .and_then(|value| value.to_str().ok()) + .filter(|value| !value.is_empty()) + else { + return false; + }; + let path = request + .uri() + .path() + .strip_prefix("/tdata/") + .unwrap_or(request.uri().path()); + let Ok(temper_odata::path::ODataPath::BoundAction { parent, action }) = + temper_odata::path::parse_path(path) + else { + return false; + }; + let temper_odata::path::ODataPath::Entity(set_name, key) = *parent else { + return false; + }; + match key { + temper_odata::path::KeyValue::Single(_) => {} + temper_odata::path::KeyValue::Composite(_) => return false, + } + let Some(entity_type) = odata::resolve_entity_type(state, tenant, &set_name) else { + return false; + }; + if !state + .bound_action_hook + .as_ref() + .is_some_and(|hook| hook.requires_generation_handoff(&entity_type, &action)) + { + return false; + } + true +} + +pub(super) fn publication_in_progress_response() -> Response { + ( + StatusCode::SERVICE_UNAVAILABLE, + axum::Json(serde_json::json!({ + "error": { + "code": "SpecPublicationInProgress", + "message": "Tenant runtime generation is being published; retry the request", + } + })), + ) + .into_response() +} diff --git a/crates/temper-server/src/router_test.rs b/crates/temper-server/src/router_test.rs index c9b7728a6..8acf123f3 100644 --- a/crates/temper-server/src/router_test.rs +++ b/crates/temper-server/src/router_test.rs @@ -4,12 +4,181 @@ use axum::http::{Request, StatusCode}; use temper_runtime::ActorSystem; use temper_runtime::tenant::TenantId; use temper_spec::csdl::parse_csdl; +use temper_store_sim::SimEventStore; use temper_store_turso::TursoEventStore; use tower::ServiceExt; use crate::events::EntityStateChange; +use crate::request_context::AgentContext; use crate::storage::StorageStack; +struct SameTenantPublicationHook; + +#[async_trait::async_trait] +impl crate::state::BoundActionHook for SameTenantPublicationHook { + fn requires_generation_handoff(&self, entity_type: &str, action: &str) -> bool { + entity_type == "Order" && action.rsplit('.').next().unwrap_or(action) == "CancelOrder" + } + + async fn after_bound_action( + &self, + ctx: crate::state::BoundActionHookContext<'_>, + ) -> Result, String> { + let publication = ctx + .state + .begin_spec_publication_after_drain( + ctx.tenant, + ctx.expected_generation.ok_or_else(|| { + "same-tenant publication hook did not receive a generation token".to_string() + })?, + ) + .await?; + drop(publication); + Ok(Some(serde_json::json!({"publicationWriter": "acquired"}))) + } +} + +struct FailOnceSameTenantPublicationHook { + attempts: std::sync::atomic::AtomicUsize, +} + +struct CountingBoundActionHook { + attempts: std::sync::atomic::AtomicUsize, + fail_first: bool, +} + +struct ReceiptFaultIdempotentHook { + store: SimEventStore, + invocations: std::sync::atomic::AtomicUsize, + external_effects: std::sync::atomic::AtomicUsize, + outputs: std::sync::Mutex>, +} + +struct InterveningGenerationHook; + +#[async_trait::async_trait] +impl crate::state::BoundActionHook for InterveningGenerationHook { + fn requires_generation_handoff(&self, entity_type: &str, action: &str) -> bool { + entity_type == "Order" && action.rsplit('.').next().unwrap_or(action) == "CancelOrder" + } + + async fn after_bound_action( + &self, + ctx: crate::state::BoundActionHookContext<'_>, + ) -> Result, String> { + let expected_generation = ctx.expected_generation.ok_or_else(|| { + "intervening-generation hook did not receive a generation token".to_string() + })?; + let mut intervening = ctx.state.begin_spec_publication(ctx.tenant).await?; + let intent = ServerState::spec_publication_intent( + "router-test-intervening-generation", + [("generation", b"new".as_slice())], + ); + ctx.state + .arm_spec_publication(&mut intervening, ctx.tenant, &intent)?; + ctx.state + .complete_spec_publication_retry(&mut intervening, ctx.tenant)?; + drop(intervening); + + ctx.state + .begin_spec_publication_after_drain(ctx.tenant, expected_generation) + .await?; + Ok(None) + } +} + +#[async_trait::async_trait] +impl crate::state::BoundActionHook for FailOnceSameTenantPublicationHook { + fn requires_generation_handoff(&self, entity_type: &str, action: &str) -> bool { + entity_type == "Order" && action.rsplit('.').next().unwrap_or(action) == "CancelOrder" + } + + async fn after_bound_action( + &self, + ctx: crate::state::BoundActionHookContext<'_>, + ) -> Result, String> { + let mut publication = ctx + .state + .begin_spec_publication_after_drain( + ctx.tenant, + ctx.expected_generation.ok_or_else(|| { + "same-tenant publication hook did not receive a generation token".to_string() + })?, + ) + .await?; + let intent = ServerState::spec_publication_intent( + "router-test-post-action", + [("generation", b"one".as_slice())], + ); + ctx.state + .arm_spec_publication(&mut publication, ctx.tenant, &intent)?; + if self + .attempts + .fetch_add(1, std::sync::atomic::Ordering::SeqCst) + == 0 + { + return Err("injected post-arm publication failure".to_string()); + } + ctx.state + .complete_spec_publication_retry(&mut publication, ctx.tenant)?; + Ok(Some(serde_json::json!({"publicationRetry": "completed"}))) + } +} + +#[async_trait::async_trait] +impl crate::state::BoundActionHook for CountingBoundActionHook { + fn requires_generation_handoff(&self, entity_type: &str, action: &str) -> bool { + entity_type == "Order" && action.rsplit('.').next().unwrap_or(action) == "CancelOrder" + } + + async fn after_bound_action( + &self, + _ctx: crate::state::BoundActionHookContext<'_>, + ) -> Result, String> { + let attempt = self + .attempts + .fetch_add(1, std::sync::atomic::Ordering::SeqCst) + + 1; + if self.fail_first && attempt == 1 { + return Err("injected first hook failure".to_string()); + } + Ok(Some(serde_json::json!({"hookAttempt": attempt}))) + } +} + +#[async_trait::async_trait] +impl crate::state::BoundActionHook for ReceiptFaultIdempotentHook { + async fn after_bound_action( + &self, + ctx: crate::state::BoundActionHookContext<'_>, + ) -> Result, String> { + let invocation = self + .invocations + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let output = { + let mut outputs = self.outputs.lock().expect("hook output lock"); + if let Some(output) = outputs.get(ctx.operation_id) { + output.clone() + } else { + self.external_effects + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let output = serde_json::json!({"operationId": ctx.operation_id}); + outputs.insert(ctx.operation_id.to_string(), output.clone()); + output + } + }; + if invocation == 0 { + self.store.restore_faults(temper_store_sim::SimFaultConfig { + write_failure_prob: 1.0, + concurrency_violation_prob: 0.0, + read_truncation_prob: 0.0, + snapshot_failure_prob: 0.0, + }); + } + Ok(Some(output)) + } +} + fn test_state() -> ServerState { let csdl_xml = include_str!("../../../test-fixtures/specs/model.csdl.xml"); let csdl = parse_csdl(csdl_xml).unwrap(); @@ -1522,6 +1691,762 @@ async fn test_post_bound_action() { assert_eq!(json["status"], "Cancelled"); } +#[tokio::test] +async fn same_tenant_post_action_publication_hands_off_request_generation() { + let mut state = test_state_with_ioa(); + state.bound_action_hook = Some(std::sync::Arc::new(SameTenantPublicationHook)); + let response = build_router(state) + .oneshot( + Request::post("/tdata/Orders('same-tenant')/Temper.Example.CancelOrder") + .header("Content-Type", "application/json") + .header("X-Temper-Principal-Kind", "admin") + .header("Idempotency-Key", "same-tenant-publication") + .body(Body::from(r#"{"Reason": "publish generation"}"#)) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), 1024 * 1024) + .await + .unwrap(); + let json: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(json["postAction"]["publicationWriter"], "acquired"); +} + +#[tokio::test] +async fn publication_context_is_the_only_actor_path_inside_an_armed_generation() { + let state = test_state_with_ioa(); + let tenant = TenantId::default(); + let intent = ServerState::spec_publication_intent( + "router-test-internal-publication", + [("generation", b"one".as_slice())], + ); + let mut publication = state + .begin_spec_publication(&tenant) + .await + .expect("acquire publication writer"); + state + .arm_spec_publication(&mut publication, &tenant, &intent) + .expect("arm publication writer"); + + assert!( + state + .get_or_spawn_tenant_actor(&tenant, "Order", "publication-owned") + .is_none(), + "unscoped actor resolution must remain fenced while publication is armed" + ); + let publication_ctx = state + .spec_publication_dispatch_context(&publication, &tenant, "router-test") + .expect("derive publication-owned dispatch context"); + state + .get_or_create_tenant_entity_in_generation( + &tenant, + "Order", + "publication-owned", + serde_json::json!({}), + &publication_ctx, + ) + .await + .expect("publication-owned actor resolution must use the live table"); + + state + .complete_spec_publication(&mut publication, &tenant) + .expect("complete first generation"); + drop(publication); + + let mut next_publication = state + .begin_spec_publication(&tenant) + .await + .expect("acquire next publication writer"); + state + .arm_spec_publication(&mut next_publication, &tenant, &intent) + .expect("arm next publication writer"); + state + .get_tenant_entity_state_in_generation( + &tenant, + "Order", + "publication-owned", + &publication_ctx, + ) + .await + .expect_err("a context from the retired generation must not respawn an actor"); + + let next_ctx = state + .spec_publication_dispatch_context(&next_publication, &tenant, "router-test") + .expect("derive next publication-owned context"); + state + .get_tenant_entity_state_in_generation(&tenant, "Order", "publication-owned", &next_ctx) + .await + .expect("the current publication generation may resolve the actor"); + state + .complete_spec_publication(&mut next_publication, &tenant) + .expect("complete next generation"); +} + +#[tokio::test] +async fn detached_work_cannot_reenter_after_its_captured_generation_retires() { + let state = test_state_with_ioa(); + let tenant = TenantId::default(); + let captured_generation = state.tenant_generation_version(&tenant); + let intent = ServerState::spec_publication_intent( + "router-test-detached-generation", + [("generation", b"next".as_slice())], + ); + let mut publication = state + .begin_spec_publication(&tenant) + .await + .expect("acquire publication writer"); + state + .arm_spec_publication(&mut publication, &tenant, &intent) + .expect("arm publication"); + assert!( + state + .try_begin_captured_tenant_generation(&tenant, captured_generation) + .await + .is_none(), + "paused detached work must not enter while publication is armed" + ); + state + .complete_spec_publication(&mut publication, &tenant) + .expect("complete replacement generation"); + drop(publication); + assert!( + state + .try_begin_captured_tenant_generation(&tenant, captured_generation) + .await + .is_none(), + "old-event work must not borrow the replacement runtime generation" + ); + let current_generation = state.tenant_generation_version(&tenant); + assert!( + state + .try_begin_captured_tenant_generation(&tenant, current_generation) + .await + .is_some(), + "work captured from the current generation remains admissible" + ); +} + +#[tokio::test] +async fn captured_generation_lease_holds_the_dispatch_boundary_against_publication() { + let state = test_state_with_ioa(); + let tenant = TenantId::default(); + let captured_generation = state.tenant_generation_version(&tenant); + let lease = state + .try_begin_captured_tenant_generation(&tenant, captured_generation) + .await + .expect("captured work should re-enter the current generation"); + let dispatch_ctx = + AgentContext::for_service("boundary-test").with_tenant_generation_lease(lease); + + let busy = match state.begin_spec_publication(&tenant).await { + Ok(_) => panic!("publication crossed an active captured dispatch generation"), + Err(error) => error, + }; + assert!(busy.contains("runtime generation is busy")); + + drop(dispatch_ctx); + let mut publication = state + .begin_spec_publication(&tenant) + .await + .expect("publisher should acquire after the dispatch context drops"); + let intent = ServerState::spec_publication_intent( + "boundary-held-generation", + [("generation", b"next".as_slice())], + ); + state + .arm_spec_publication(&mut publication, &tenant, &intent) + .expect("arm replacement generation"); + state + .complete_spec_publication(&mut publication, &tenant) + .expect("complete replacement generation"); +} + +#[tokio::test] +async fn immediate_generation_fork_drains_before_publication_cutover() { + let state = test_state_with_ioa(); + let tenant = TenantId::default(); + let generation = state.tenant_generation_version(&tenant); + let request = state + .try_begin_captured_tenant_generation(&tenant, generation) + .await + .expect("capture request generation"); + let immediate = request + .fork_immediate(&tenant) + .expect("fork immediate obligation"); + request.release(); + + let busy = match state.begin_spec_publication(&tenant).await { + Ok(_) => panic!("publication must drain the independently forked obligation"), + Err(error) => error, + }; + assert!(busy.contains("runtime generation is busy")); + + drop(immediate); + state + .begin_spec_publication(&tenant) + .await + .expect("publication may begin after the immediate obligation completes"); +} + +#[tokio::test] +async fn released_request_lease_cannot_borrow_a_publication_gate() { + let state = test_state_with_ioa(); + let tenant = TenantId::default(); + let generation = state.tenant_generation_version(&tenant); + let lease = state + .try_begin_captured_tenant_generation(&tenant, generation) + .await + .expect("capture request generation"); + let agent_ctx = AgentContext::for_service("released-request-test") + .with_tenant_generation_lease(lease.clone()); + lease.release(); + + let mut publication = state + .begin_spec_publication(&tenant) + .await + .expect("acquire publication writer after request release"); + let intent = ServerState::spec_publication_intent( + "released-request-provenance", + [("generation", b"pending".as_slice())], + ); + state + .arm_spec_publication(&mut publication, &tenant, &intent) + .expect("arm pending generation"); + + let error = state + .dispatch_tenant_action( + &tenant, + "Order", + "released-request", + "AddItem", + serde_json::json!({}), + &agent_ctx, + ) + .await + .expect_err("released request provenance must not dispatch inside a writer gate"); + assert!( + error.contains("dispatch deferred"), + "unexpected error: {error}" + ); +} + +#[tokio::test] +async fn publication_owned_background_work_waits_for_the_pending_generation() { + let state = test_state_with_ioa(); + let tenant = TenantId::default(); + let pending_generation = state + .tenant_generation_version(&tenant) + .checked_add(1) + .expect("test generation should not overflow"); + let mut publication = state + .begin_spec_publication(&tenant) + .await + .expect("acquire publication writer"); + let intent = ServerState::spec_publication_intent( + "queued-publication-work", + [("generation", b"next".as_slice())], + ); + state + .arm_spec_publication(&mut publication, &tenant, &intent) + .expect("arm pending generation"); + + let queued_state = state.clone(); + let queued_tenant = tenant.clone(); + let (finished_tx, mut finished_rx) = tokio::sync::oneshot::channel(); + let queued = tokio::spawn(async move { + let lease = queued_state + .begin_captured_tenant_generation(&queued_tenant, pending_generation) + .await; + let _ = finished_tx.send(lease.is_some()); + }); + tokio::task::yield_now().await; + assert!( + matches!( + finished_rx.try_recv(), + Err(tokio::sync::oneshot::error::TryRecvError::Empty) + ), + "publication-owned effects must wait behind the writer boundary" + ); + + state + .complete_spec_publication(&mut publication, &tenant) + .expect("complete pending generation"); + drop(publication); + assert!( + finished_rx + .await + .expect("queued background task should report its admission"), + "queued work should enter the exact newly published generation" + ); + queued.await.expect("queued task should finish"); +} + +#[tokio::test] +async fn sticky_debt_retry_cannot_enter_while_an_exact_retry_writer_is_active() { + let state = test_state_with_ioa(); + let tenant = TenantId::default(); + let intent = ServerState::spec_publication_intent( + "sticky-debt-retry-race", + [("generation", b"same".as_slice())], + ); + let mut ambiguous = state + .begin_spec_publication(&tenant) + .await + .expect("acquire first publication writer"); + state + .arm_spec_publication(&mut ambiguous, &tenant, &intent) + .expect("arm first publication"); + drop(ambiguous); + assert!(state.spec_publication_gated(&tenant)); + + let first_retry_reader = state + .try_begin_tenant_request(&tenant) + .await + .expect("idle sticky debt should admit one stable retry reader"); + drop(first_retry_reader); + let mut active_retry = state + .begin_spec_publication(&tenant) + .await + .expect("exact retry should acquire the writer after handoff"); + state + .arm_spec_publication(&mut active_retry, &tenant, &intent) + .expect("exact retry should inherit the same debt"); + + assert!( + state.try_begin_tenant_request(&tenant).await.is_none(), + "a second retry must not read registry, Cedar, or actors during the active cutover" + ); + state + .complete_spec_publication_retry(&mut active_retry, &tenant) + .expect("exact retry should discharge sticky debt"); +} + +#[tokio::test] +async fn same_tenant_post_action_rejects_an_intervening_generation() { + let mut state = test_state_with_ioa(); + state.bound_action_hook = Some(std::sync::Arc::new(InterveningGenerationHook)); + let tenant = TenantId::default(); + let response = build_router(state.clone()) + .oneshot( + Request::post("/tdata/Orders('generation-race')/Temper.Example.CancelOrder") + .header("Content-Type", "application/json") + .header("X-Temper-Principal-Kind", "admin") + .header("Idempotency-Key", "generation-race-publication") + .body(Body::from(r#"{"Reason": "publish generation"}"#)) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!(state.tenant_generation_version(&tenant), 1); +} + +#[tokio::test] +async fn publication_capable_action_requires_idempotency_before_transition() { + let mut state = test_state_with_ioa(); + state.bound_action_hook = Some(std::sync::Arc::new(SameTenantPublicationHook)); + let tenant = TenantId::default(); + let response = build_router(state.clone()) + .oneshot( + Request::post("/tdata/Orders('keyless-publication')/Temper.Example.CancelOrder") + .header("Content-Type", "application/json") + .header("X-Temper-Principal-Kind", "admin") + .body(Body::from(r#"{"Reason": "must not transition"}"#)) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert!(!state.entity_exists(&tenant, "Order", "keyless-publication")); + assert!(!state.spec_publication_gated(&tenant)); +} + +#[tokio::test] +async fn unproved_actor_cache_entry_cannot_be_rebound_to_bound_action() { + let state = test_state_with_ioa(); + let tenant = TenantId::default(); + let seeded = state + .get_or_create_tenant_entity(&tenant, "Order", "unproved-cache", serde_json::json!({})) + .await + .unwrap(); + state + .idempotency_cache + .put("default:Order:unproved-cache", "shared-raw-key", seeded); + + let response = build_router(state) + .oneshot( + Request::post("/tdata/Orders('unproved-cache')/Temper.Example.CancelOrder") + .header("Content-Type", "application/json") + .header("X-Temper-Principal-Kind", "admin") + .header("Idempotency-Key", "shared-raw-key") + .body(Body::from(r#"{"Reason": "must conflict"}"#)) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(response.status(), StatusCode::CONFLICT); +} + +#[tokio::test] +async fn concurrent_bound_actions_cannot_rebind_one_raw_idempotency_key() { + let app = build_router(test_state_with_ioa()); + let first = app.clone().oneshot( + Request::post("/tdata/Orders('concurrent-key')/Temper.Example.CancelOrder") + .header("Content-Type", "application/json") + .header("X-Temper-Principal-Kind", "admin") + .header("Idempotency-Key", "concurrent-raw-key") + .body(Body::from(r#"{"Reason": "first"}"#)) + .unwrap(), + ); + let second = app.oneshot( + Request::post("/tdata/Orders('concurrent-key')/Temper.Example.CancelOrder") + .header("Content-Type", "application/json") + .header("X-Temper-Principal-Kind", "admin") + .header("Idempotency-Key", "concurrent-raw-key") + .body(Body::from(r#"{"Reason": "second"}"#)) + .unwrap(), + ); + let (first, second) = tokio::join!(first, second); + let mut statuses = [first.unwrap().status(), second.unwrap().status()]; + statuses.sort(); + + assert_eq!(statuses, [StatusCode::OK, StatusCode::CONFLICT]); +} + +#[tokio::test] +async fn exact_bound_action_retry_reuses_the_completed_post_action_output() { + let mut state = test_state_with_ioa(); + let hook = std::sync::Arc::new(CountingBoundActionHook { + attempts: std::sync::atomic::AtomicUsize::new(0), + fail_first: false, + }); + state.bound_action_hook = Some(hook.clone()); + let app = build_router(state); + let request = || { + Request::post("/tdata/Orders('hook-once')/Temper.Example.CancelOrder") + .header("Content-Type", "application/json") + .header("X-Temper-Principal-Kind", "admin") + .header("Idempotency-Key", "hook-once-key") + .body(Body::from(r#"{"Reason": "one hook"}"#)) + .unwrap() + }; + + let first = app.clone().oneshot(request()).await.unwrap(); + let retry = app.oneshot(request()).await.unwrap(); + assert_eq!(first.status(), StatusCode::OK); + assert_eq!(retry.status(), StatusCode::OK); + let first: serde_json::Value = serde_json::from_slice( + &axum::body::to_bytes(first.into_body(), 1024 * 1024) + .await + .unwrap(), + ) + .unwrap(); + let retry: serde_json::Value = serde_json::from_slice( + &axum::body::to_bytes(retry.into_body(), 1024 * 1024) + .await + .unwrap(), + ) + .unwrap(); + assert_eq!(retry["postAction"], first["postAction"]); + assert_eq!( + hook.attempts.load(std::sync::atomic::Ordering::SeqCst), + 1, + "a completed post-action hook must not run again on an exact retry" + ); +} + +#[tokio::test] +async fn completed_bound_action_hook_survives_cache_loss_and_actor_respawn() { + let mut state = test_state_with_ioa(); + state.set_storage_stack(StorageStack::from_sim(SimEventStore::no_faults(409), None)); + let hook = std::sync::Arc::new(CountingBoundActionHook { + attempts: std::sync::atomic::AtomicUsize::new(0), + fail_first: false, + }); + state.bound_action_hook = Some(hook.clone()); + let app = build_router(state.clone()); + let request = || { + Request::post("/tdata/Orders('hook-restart')/Temper.Example.CancelOrder") + .header("Content-Type", "application/json") + .header("X-Temper-Principal-Kind", "admin") + .header("Idempotency-Key", "hook-restart-key") + .body(Body::from(r#"{"Reason": "durable hook"}"#)) + .unwrap() + }; + + let first = app.clone().oneshot(request()).await.unwrap(); + assert_eq!(first.status(), StatusCode::OK); + let first: serde_json::Value = serde_json::from_slice( + &axum::body::to_bytes(first.into_body(), 1024 * 1024) + .await + .unwrap(), + ) + .unwrap(); + + let actor_key = "default:Order:hook-restart"; + state.idempotency_cache.clear_actor_for_test(actor_key); + state + .last_accessed + .write() + .expect("last-accessed lock") + .insert( + actor_key.to_string(), + temper_runtime::scheduler::sim_now() - chrono::Duration::seconds(600), + ); + state.passivate_idle_actors().await; + assert!( + !state + .actor_registry + .read() + .expect("actor registry lock") + .contains_key(actor_key), + "fixture must remove the live actor before retry" + ); + + let retry = app.oneshot(request()).await.unwrap(); + assert_eq!(retry.status(), StatusCode::OK); + let retry: serde_json::Value = serde_json::from_slice( + &axum::body::to_bytes(retry.into_body(), 1024 * 1024) + .await + .unwrap(), + ) + .unwrap(); + assert_eq!(retry["postAction"], first["postAction"]); + assert_eq!( + hook.attempts.load(std::sync::atomic::Ordering::SeqCst), + 1, + "a completed durable hook must not execute again after actor respawn" + ); +} + +#[tokio::test] +async fn pending_hook_receipt_replays_one_content_idempotent_operation_after_respawn() { + let store = SimEventStore::no_faults(419); + let mut state = test_state_with_ioa(); + state.set_storage_stack(StorageStack::from_sim(store.clone(), None)); + let hook = std::sync::Arc::new(ReceiptFaultIdempotentHook { + store: store.clone(), + invocations: std::sync::atomic::AtomicUsize::new(0), + external_effects: std::sync::atomic::AtomicUsize::new(0), + outputs: std::sync::Mutex::new(std::collections::BTreeMap::new()), + }); + state.bound_action_hook = Some(hook.clone()); + let app = build_router(state.clone()); + let request = || { + Request::post("/tdata/Orders('hook-receipt-fault')/Temper.Example.CancelOrder") + .header("Content-Type", "application/json") + .header("X-Temper-Principal-Kind", "admin") + .header("Idempotency-Key", "hook-receipt-fault-key") + .body(Body::from(r#"{"Reason": "receipt failure"}"#)) + .unwrap() + }; + + assert_eq!( + app.clone().oneshot(request()).await.unwrap().status(), + StatusCode::INTERNAL_SERVER_ERROR, + "the hook commits before its injected completion-receipt failure" + ); + store.disable_faults(); + + let actor_key = "default:Order:hook-receipt-fault"; + state.idempotency_cache.clear_actor_for_test(actor_key); + state + .last_accessed + .write() + .expect("last-accessed lock") + .insert( + actor_key.to_string(), + temper_runtime::scheduler::sim_now() - chrono::Duration::seconds(600), + ); + state.passivate_idle_actors().await; + + let recovered = app.clone().oneshot(request()).await.unwrap(); + let exact_retry = app.oneshot(request()).await.unwrap(); + assert_eq!(recovered.status(), StatusCode::OK); + assert_eq!(exact_retry.status(), StatusCode::OK); + let recovered: serde_json::Value = serde_json::from_slice( + &axum::body::to_bytes(recovered.into_body(), 1024 * 1024) + .await + .unwrap(), + ) + .unwrap(); + let exact_retry: serde_json::Value = serde_json::from_slice( + &axum::body::to_bytes(exact_retry.into_body(), 1024 * 1024) + .await + .unwrap(), + ) + .unwrap(); + assert_eq!(exact_retry["postAction"], recovered["postAction"]); + assert_eq!( + hook.invocations.load(std::sync::atomic::Ordering::SeqCst), + 2, + "the pending durable intent is retried exactly once" + ); + assert_eq!( + hook.external_effects + .load(std::sync::atomic::Ordering::SeqCst), + 1, + "the stable operation ID deduplicates the external mutation" + ); +} + +#[tokio::test] +async fn failed_bound_action_hook_retries_once_then_caches_its_success() { + let mut state = test_state_with_ioa(); + let hook = std::sync::Arc::new(CountingBoundActionHook { + attempts: std::sync::atomic::AtomicUsize::new(0), + fail_first: true, + }); + state.bound_action_hook = Some(hook.clone()); + let app = build_router(state); + let request = || { + Request::post("/tdata/Orders('hook-retry')/Temper.Example.CancelOrder") + .header("Content-Type", "application/json") + .header("X-Temper-Principal-Kind", "admin") + .header("Idempotency-Key", "hook-retry-key") + .body(Body::from(r#"{"Reason": "retry hook"}"#)) + .unwrap() + }; + + assert_eq!( + app.clone().oneshot(request()).await.unwrap().status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + let recovered = app.clone().oneshot(request()).await.unwrap(); + let retry = app.oneshot(request()).await.unwrap(); + assert_eq!(recovered.status(), StatusCode::OK); + assert_eq!(retry.status(), StatusCode::OK); + let recovered: serde_json::Value = serde_json::from_slice( + &axum::body::to_bytes(recovered.into_body(), 1024 * 1024) + .await + .unwrap(), + ) + .unwrap(); + let retry: serde_json::Value = serde_json::from_slice( + &axum::body::to_bytes(retry.into_body(), 1024 * 1024) + .await + .unwrap(), + ) + .unwrap(); + assert_eq!(retry["postAction"], recovered["postAction"]); + assert_eq!( + hook.attempts.load(std::sync::atomic::Ordering::SeqCst), + 2, + "a successful hook retry must become the cached terminal result" + ); +} + +#[tokio::test] +async fn gated_idempotent_post_action_retry_completes_exact_publication() { + let mut state = test_state_with_ioa(); + state.set_storage_stack(StorageStack::from_sim(SimEventStore::no_faults(401), None)); + state.bound_action_hook = Some(std::sync::Arc::new(FailOnceSameTenantPublicationHook { + attempts: std::sync::atomic::AtomicUsize::new(0), + })); + let tenant = TenantId::default(); + let app = build_router(state.clone()); + let request = || { + Request::post("/tdata/Orders('gated-retry')/Temper.Example.CancelOrder") + .header("Content-Type", "application/json") + .header("X-Temper-Principal-Kind", "admin") + .header("X-Temper-Principal-Id", "original-operator") + .header("X-Session-Id", "publication-session-one") + .header("Idempotency-Key", "gated-retry-1") + .body(Body::from(r#"{"Reason": "retry publication"}"#)) + .unwrap() + }; + + let failed = app.clone().oneshot(request()).await.unwrap(); + assert_eq!(failed.status(), StatusCode::INTERNAL_SERVER_ERROR); + assert!(state.spec_publication_gated(&tenant)); + state + .idempotency_cache + .clear_actor_for_test("default:Order:gated-retry"); + + let changed_params = app + .clone() + .oneshot( + Request::post("/tdata/Orders('gated-retry')/Temper.Example.CancelOrder") + .header("Content-Type", "application/json") + .header("X-Temper-Principal-Kind", "admin") + .header("X-Temper-Principal-Id", "other-operator") + .header("Idempotency-Key", "gated-retry-1") + .body(Body::from(r#"{"Reason": "different target"}"#)) + .unwrap(), + ) + .await + .unwrap(); + let changed_params_status = changed_params.status(); + let changed_params_body = axum::body::to_bytes(changed_params.into_body(), 1024 * 1024) + .await + .unwrap(); + assert_eq!( + changed_params_status, + StatusCode::CONFLICT, + "unexpected changed-params response: {}", + String::from_utf8_lossy(&changed_params_body) + ); + assert!(state.spec_publication_gated(&tenant)); + + let changed_principal = app + .clone() + .oneshot( + Request::post("/tdata/Orders('gated-retry')/Temper.Example.CancelOrder") + .header("Content-Type", "application/json") + .header("X-Temper-Principal-Kind", "customer") + .header("X-Temper-Principal-Id", "different-principal") + .header("Idempotency-Key", "gated-retry-1") + .body(Body::from(r#"{"Reason": "retry publication"}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(changed_principal.status(), StatusCode::CONFLICT); + assert!(state.spec_publication_gated(&tenant)); + + let changed_action = app + .clone() + .oneshot( + Request::post("/tdata/Orders('gated-retry')/Temper.Example.InitiateReturn") + .header("Content-Type", "application/json") + .header("X-Temper-Principal-Kind", "admin") + .header("Idempotency-Key", "gated-retry-1") + .body(Body::from(r#"{"Reason": "retry publication"}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(changed_action.status(), StatusCode::SERVICE_UNAVAILABLE); + assert!(state.spec_publication_gated(&tenant)); + + let recovered = app + .oneshot( + Request::post("/tdata/Orders('gated-retry')/Temper.Example.CancelOrder") + .header("Content-Type", "application/json") + .header("X-Temper-Principal-Kind", "admin") + .header("X-Temper-Principal-Id", "recovery-operator") + .header("X-Session-Id", "publication-session-two") + .header("Idempotency-Key", "gated-retry-1") + .body(Body::from(r#"{"Reason": "retry publication"}"#)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(recovered.status(), StatusCode::OK); + let body = axum::body::to_bytes(recovered.into_body(), 1024 * 1024) + .await + .unwrap(); + let json: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(json["postAction"]["publicationRetry"], "completed"); + assert!(!state.spec_publication_gated(&tenant)); +} + #[tokio::test] async fn test_odata_version_header() { let app = build_router(test_state()); diff --git a/crates/temper-server/src/state/custom_effects.rs b/crates/temper-server/src/state/custom_effects.rs index de7600979..42f00ae2e 100644 --- a/crates/temper-server/src/state/custom_effects.rs +++ b/crates/temper-server/src/state/custom_effects.rs @@ -12,6 +12,10 @@ /// /// Implementations registered on [`ServerState::custom_effect_handler`] are /// called by the post-dispatch effects pipeline for every custom effect name. +/// The journal may replay a handler after an outcome-ambiguous failure, so a +/// handler must make external mutations idempotent for the supplied entity +/// identity and effect input. +#[async_trait::async_trait] pub trait CustomEffectHandler: Send + Sync { /// Handle a custom effect. /// @@ -20,7 +24,7 @@ pub trait CustomEffectHandler: Send + Sync { /// - `entity_id`: the entity instance ID /// - `entity_fields`: merged entity state fields (includes all params from all prior actions) /// - `server`: server state for dispatch, authz, etc. - fn handle( + async fn handle( &self, effect_name: &str, entity_type: &str, diff --git a/crates/temper-server/src/state/dispatch/actions.rs b/crates/temper-server/src/state/dispatch/actions.rs index 20954e510..0d2536969 100644 --- a/crates/temper-server/src/state/dispatch/actions.rs +++ b/crates/temper-server/src/state/dispatch/actions.rs @@ -37,6 +37,35 @@ fn background_reaction_semaphore() -> Arc { } impl crate::state::ServerState { + pub(super) async fn dispatch_context_in_generation( + &self, + tenant: &TenantId, + agent_ctx: &AgentContext, + ) -> Result, DispatchError> { + if let Some(lease) = agent_ctx.tenant_generation_lease.as_ref() + && lease.belongs_to(tenant) + && lease.captured_generation() == self.tenant_generation_version(tenant) + && (lease.is_held_for(tenant) + || (lease.is_publication_owned_for(tenant) && self.spec_publication_gated(tenant))) + { + return Ok(Box::new(agent_ctx.clone())); + } + if self.spec_publication_gated(tenant) { + return Err(DispatchError::Deferred { retry_after_ms: 1 }); + } + let generation = self + .try_begin_tenant_request(tenant) + .await + .ok_or(DispatchError::Deferred { retry_after_ms: 1 })?; + if self.spec_publication_gated(tenant) { + return Err(DispatchError::Deferred { retry_after_ms: 1 }); + } + let captured_generation = self.tenant_generation_version(tenant); + Ok(Box::new(agent_ctx.clone().with_tenant_generation_lease( + crate::state::TenantGenerationLease::new(tenant, generation, captured_generation), + ))) + } + /// Dispatch an action using the unified command object. /// /// This is the preferred entry point. The command struct makes all @@ -162,6 +191,10 @@ impl crate::state::ServerState { await_integration, await_reactions, } = cmd; + let dispatch_agent_ctx = self + .dispatch_context_in_generation(tenant, agent_ctx) + .await?; + let agent_ctx = dispatch_agent_ctx.as_ref(); if self .composite_metadata_for(tenant, entity_type, action)? @@ -272,6 +305,13 @@ impl crate::state::ServerState { fields, agent_ctx, } = dispatch; + let agent_ctx = agent_ctx + .fork_tenant_generation_lease(&tenant) + .unwrap_or_else(|| { + let mut detached = agent_ctx; + detached.detach_tenant_generation_lease(); + detached + }); let span = tracing::info_span!( "reaction.dispatch.background", tenant = %tenant, @@ -282,6 +322,19 @@ impl crate::state::ServerState { let reaction_task = async move { let _permit = permit; + let Some(agent_ctx) = state + .activate_immediate_tenant_work(&tenant, agent_ctx) + .await + else { + tracing::error!( + tenant = %tenant, + entity_type = %entity_type, + entity_id = %entity_id, + action_name = %action, + "background reactions could not enter a complete runtime generation" + ); + return; + }; let results = dispatcher .dispatch_reactions( &state, @@ -339,6 +392,10 @@ impl crate::state::ServerState { agent_ctx: &AgentContext, await_integration: bool, ) -> Result { + let generation_agent_ctx = self + .dispatch_context_in_generation(tenant, agent_ctx) + .await?; + let agent_ctx = generation_agent_ctx.as_ref(); let explicit_workflow_context = agent_ctx.workflow_run_id.is_some() || agent_ctx.workflow_root_entity_type.is_some() || agent_ctx.workflow_root_entity_id.is_some(); @@ -424,7 +481,12 @@ impl crate::state::ServerState { .read() .map(|reg| reg.contains_key(&actor_key)) .unwrap_or(false); - let Some(ar) = self.get_or_spawn_tenant_actor(tenant, entity_type, entity_id) else { + let Some(ar) = self.get_or_spawn_tenant_actor_in_generation( + tenant, + entity_type, + entity_id, + agent_ctx, + ) else { return Err(DispatchError::Internal(format!( "failed to resolve actor for governed entity type '{entity_type}'" ))); @@ -606,6 +668,23 @@ impl crate::state::ServerState { let response = match outcome.result { Ok(response) => response, Err(e) => { + if !actor_existed_before + && matches!( + &e, + temper_runtime::actor::ActorError::Stopped + | temper_runtime::actor::ActorError::SendFailed + | temper_runtime::actor::ActorError::InitFailed(_) + | temper_runtime::actor::ActorError::MaxRestartsExceeded(_) + ) + { + self.discard_uncommitted_spawn_after_dispatch_failure( + tenant, + entity_type, + entity_id, + actor_ref.id(), + ) + .await; + } let entry = TrajectoryEntry { timestamp: sim_now().to_rfc3339(), tenant: tenant.to_string(), diff --git a/crates/temper-server/src/state/dispatch/adapter.rs b/crates/temper-server/src/state/dispatch/adapter.rs index e26b7dd40..be425472c 100644 --- a/crates/temper-server/src/state/dispatch/adapter.rs +++ b/crates/temper-server/src/state/dispatch/adapter.rs @@ -38,7 +38,14 @@ impl crate::state::ServerState { let action = input.action.to_string(); let custom_effects = input.custom_effects.to_vec(); let entity_state = input.entity_state.clone(); - let agent_ctx = input.agent_ctx.clone(); + let agent_ctx = input + .agent_ctx + .fork_tenant_generation_lease(input.tenant) + .unwrap_or_else(|| { + let mut detached = input.agent_ctx.clone(); + detached.detach_tenant_generation_lease(); + detached + }); let action_params = input.action_params.clone(); let workflow_root_entity_type = agent_ctx .workflow_root_entity_type @@ -65,6 +72,18 @@ impl crate::state::ServerState { tokio::spawn( async move { // determinism-ok: async integration side-effects run outside simulation core + let Some(agent_ctx) = state + .activate_immediate_tenant_work(&tenant, agent_ctx) + .await + else { + tracing::error!( + tenant = %tenant, + entity_type = %entity_type, + entity_id = %entity_id, + "adapter integrations could not enter a complete runtime generation" + ); + return; + }; let req = WasmDispatchRequest { tenant: &tenant, entity_type: &entity_type, @@ -88,6 +107,7 @@ impl crate::state::ServerState { &entity_id, &action, &e, + &agent_ctx, ); } } @@ -112,8 +132,12 @@ impl crate::state::ServerState { &self, req: &WasmDispatchRequest<'_>, ) -> Result, String> { + let generation_agent_ctx = self + .dispatch_context_in_generation(req.tenant, req.agent_ctx) + .await + .map_err(|error| error.to_string())?; record_workflow_span_attrs( - req.agent_ctx, + generation_agent_ctx.as_ref(), req.entity_type, req.entity_id, Some(req.action), @@ -136,7 +160,7 @@ impl crate::state::ServerState { entity_id: req.entity_id, }, action: req.action, - agent_ctx: req.agent_ctx, + agent_ctx: generation_agent_ctx.as_ref(), mode: req.mode, }; diff --git a/crates/temper-server/src/state/dispatch/compensation.rs b/crates/temper-server/src/state/dispatch/compensation.rs index 3d1e028fd..9b006c5ff 100644 --- a/crates/temper-server/src/state/dispatch/compensation.rs +++ b/crates/temper-server/src/state/dispatch/compensation.rs @@ -51,6 +51,7 @@ impl crate::state::ServerState { entity_id: &str, triggering_action: &str, error: &str, + agent_ctx: &AgentContext, ) { let state = self.clone(); let tenant = tenant.clone(); @@ -58,6 +59,13 @@ impl crate::state::ServerState { let entity_id = entity_id.to_string(); let triggering_action = triggering_action.to_string(); let error = error.to_string(); + let agent_ctx = agent_ctx + .fork_tenant_generation_lease(&tenant) + .unwrap_or_else(|| { + let mut detached = agent_ctx.clone(); + detached.detach_tenant_generation_lease(); + detached + }); let span = tracing::info_span!( "dispatch.integration_failure_compensation", tenant = %tenant, @@ -78,6 +86,7 @@ impl crate::state::ServerState { &entity_id, &triggering_action, &error, + agent_ctx, ) .await; } @@ -94,7 +103,21 @@ impl crate::state::ServerState { entity_id: &str, triggering_action: &str, error: &str, + agent_ctx: AgentContext, ) { + let Some(generation_ctx) = self.activate_immediate_tenant_work(tenant, agent_ctx).await + else { + self.surface_dropped_integration_failure( + tenant, + entity_type, + entity_id, + triggering_action, + "", + error, + "integration compensation could not enter a complete runtime generation", + ); + return; + }; let status = self .resolve_entity_status(tenant, entity_type, entity_id) .await @@ -122,7 +145,8 @@ impl crate::state::ServerState { "error_message": error, "trigger_action": triggering_action, }); - let agent_ctx = AgentContext::for_service("integration-compensation"); + let agent_ctx = + AgentContext::for_service_inheriting("integration-compensation", &generation_ctx); match self .dispatch_tenant_action(tenant, entity_type, entity_id, &action, params, &agent_ctx) .await diff --git a/crates/temper-server/src/state/dispatch/composite.rs b/crates/temper-server/src/state/dispatch/composite.rs index 554a26a59..dbc9b7079 100644 --- a/crates/temper-server/src/state/dispatch/composite.rs +++ b/crates/temper-server/src/state/dispatch/composite.rs @@ -8,15 +8,18 @@ use temper_authz::SecurityContext; use temper_jit::table::{CompositeActionMetadata, TransitionTable}; use temper_runtime::persistence::{ COMPOSITE_EVENT_TYPE, CompositeEvent, CompositeEventSubWrite, EventMetadata, PersistenceAppend, - PersistenceEnvelope, PersistenceError, + PersistenceBatchIdempotency, PersistenceEnvelope, PersistenceError, SnapshotSourceFence, }; use temper_runtime::scheduler::{sim_now, sim_uuid}; use temper_runtime::tenant::TenantId; -use crate::entity_actor::EntityState; use crate::entity_actor::effects::{ FieldSyncMode, build_eval_context_with_xref, process_action_with_xref_and_field_mode, }; +use crate::entity_actor::{ + EntityRecoveryContext, EntityResponse, EntityState, recover_entity_state_from_stable_sources, + state_materialization_envelope, +}; use crate::request_context::AgentContext; use crate::state::account_verification::CommonsAccountVerificationError; use crate::state::app_uniqueness::CommonsAppUniquenessError; @@ -24,11 +27,16 @@ use crate::state::storage_caps::{CommonsStorageCapError, CommonsStorageWrite}; use crate::storage::BackendLabel; use super::DispatchError; +use super::effects::PostDispatchContext; +mod atomic; mod helpers; +mod preflight; mod projection; use helpers::*; +const MAX_ACTIVE_COMPOSITE_CLAIM_LOCKS: usize = 4_096; + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct CompositeSubWrite { #[serde(alias = "target_entity", alias = "EntityType", alias = "entity")] @@ -52,6 +60,7 @@ struct PreparedCompositeSubWrite { uses_parent_gate: bool, } +#[derive(Clone, Copy)] struct AtomicCompositeParent<'a> { tenant: &'a TenantId, entity_type: &'a str, @@ -59,6 +68,16 @@ struct AtomicCompositeParent<'a> { action: &'a str, idempotency: &'a str, record_event: bool, + agent_ctx: &'a AgentContext, +} + +struct AtomicCompositePostCommit { + entity_type: String, + entity_id: String, + action: String, + params: Value, + idempotency_key: String, + response: EntityResponse, } #[derive(Debug, Clone)] @@ -71,9 +90,13 @@ struct PreflightCompositeTarget { struct AtomicCompositeStream { entity_type: String, entity_id: String, + /// Exact transition table captured before any composite recovery/evaluation. + table: TransitionTable, target_existed: bool, state: EntityState, expected_sequence: u64, + snapshot_source: SnapshotSourceFence, + materialization_baseline: Option, events: Vec, } @@ -87,7 +110,233 @@ fn empty_params() -> Value { Value::Object(Default::default()) } +fn prepare_composite_intent_sub_writes( + parent: AtomicCompositeParent<'_>, + sub_writes: &[CompositeSubWrite], + metadata: &CompositeActionMetadata, +) -> Vec { + sub_writes + .iter() + .cloned() + .enumerate() + .map(|(idx, sub_write)| { + let entity_type = sub_write.entity_type.clone(); + let entity_id = sub_write.entity_id.clone(); + let action = sub_write.action.clone(); + PreparedCompositeSubWrite { + idx, + entity_type: entity_type.clone(), + entity_id, + action: action.clone(), + params: normalize_sub_write_params(sub_write), + idempotency_key: format!( + "composite:{}:{}:{}:{}:{}:subwrite:{idx}", + parent.tenant, + parent.entity_type, + parent.entity_id, + parent.action, + parent.idempotency + ), + preflight_target: None, + uses_parent_gate: composite_sub_write_uses_parent_gate( + metadata, + &entity_type, + &action, + ), + } + }) + .collect() +} + +fn composite_batch_claim( + parent: AtomicCompositeParent<'_>, + prepared_sub_writes: &[PreparedCompositeSubWrite], +) -> Result { + let persistence_id = format!( + "{}:{}:{}", + parent.tenant, parent.entity_type, parent.entity_id + ); + let composite_event = build_composite_event( + parent.tenant, + parent.entity_type, + parent.entity_id, + parent.action, + parent.idempotency, + prepared_sub_writes, + ); + let intent_sub_writes = prepared_sub_writes + .iter() + .map(|write| { + ( + write.idx, + write.entity_type.as_str(), + write.entity_id.as_str(), + write.action.as_str(), + canonical_json(&write.params), + write.idempotency_key.as_str(), + write.uses_parent_gate, + ) + }) + .collect::>(); + let intent_bytes = + serde_json::to_vec(&(parent.record_event, composite_event, intent_sub_writes)).map_err( + |error| { + DispatchError::Internal(format!( + "failed to encode composite batch idempotency intent: {error}" + )) + }, + )?; + let intent_hash = format!("{:x}", Sha256::digest(intent_bytes)); + let repairs_incomplete_pack_payload = prepared_sub_writes.iter().any(|write| { + write.uses_parent_gate + && write.action == "Create" + && is_pack_object_entity(&write.entity_type) + && !has_complete_git_object_payload(&write.params) + }); + let idempotency_key = if repairs_incomplete_pack_payload { + format!("{}:partial:{intent_hash}", parent.idempotency) + } else { + parent.idempotency.to_string() + }; + Ok(PersistenceBatchIdempotency { + persistence_id, + idempotency_key, + intent_hash, + }) +} + +fn composite_claim_lock_key(claim: &PersistenceBatchIdempotency) -> String { + debug_assert!(!claim.persistence_id.is_empty()); + debug_assert!(!claim.idempotency_key.is_empty()); + format!( + "{}:{}{}:{}", + claim.persistence_id.len(), + claim.persistence_id, + claim.idempotency_key.len(), + claim.idempotency_key + ) +} + impl crate::state::ServerState { + fn effectful_composite_reservation_persistence_id( + parent: AtomicCompositeParent<'_>, + intended: &CompositeEvent, + ) -> String { + let mut digest = Sha256::new(); + for component in [ + parent.entity_type, + parent.entity_id, + intended.composite_idempotency_key.as_str(), + ] { + digest.update((component.len() as u64).to_be_bytes()); + digest.update(component.as_bytes()); + } + format!("{}:_CompositeIntent:{:x}", parent.tenant, digest.finalize()) + } + + async fn effectful_composite_reservation_exists( + &self, + store: &crate::storage::BoxedEventStore, + parent: AtomicCompositeParent<'_>, + intended: &CompositeEvent, + ) -> Result { + let persistence_id = Self::effectful_composite_reservation_persistence_id(parent, intended); + let events = store + .read_events(&persistence_id, 0) + .await + .map_err(composite_batch_persistence_error)?; + let mut existing: Option = None; + for event in events { + if event.event_type != COMPOSITE_EVENT_TYPE { + continue; + } + let decoded = + serde_json::from_value::(event.payload).map_err(|error| { + DispatchError::Internal(format!( + "malformed composite reservation at sequence {}: {error}", + event.sequence_nr + )) + })?; + if decoded.composite_idempotency_key != intended.composite_idempotency_key { + continue; + } + if existing.as_ref().is_some_and(|prior| prior != &decoded) { + return Err(DispatchError::Internal(format!( + "composite idempotency key '{}' has conflicting durable reservations", + parent.idempotency + ))); + } + existing = Some(decoded); + } + let Some(existing) = existing else { + return Ok(false); + }; + if existing != *intended { + return Err(DispatchError::Internal(format!( + "composite idempotency key '{}' was reused with a different intent", + parent.idempotency + ))); + } + Ok(true) + } + + async fn persist_effectful_composite_reservation( + &self, + store: &crate::storage::BoxedEventStore, + parent: AtomicCompositeParent<'_>, + intended: &CompositeEvent, + ) -> Result<(), DispatchError> { + if self + .effectful_composite_reservation_exists(store, parent, intended) + .await? + { + return Ok(()); + } + let persistence_id = Self::effectful_composite_reservation_persistence_id(parent, intended); + let boundary = store + .journal_boundary(&persistence_id) + .await + .map_err(composite_batch_persistence_error)?; + let envelope = composite_event_envelope(&persistence_id, intended)?; + if let Err(error) = store + .append(&persistence_id, boundary.latest_sequence, &[envelope]) + .await + && !self + .effectful_composite_reservation_exists(store, parent, intended) + .await? + { + return Err(composite_batch_persistence_error(error)); + } + + Ok(()) + } + + async fn acquire_composite_claim_lock( + &self, + claim: &PersistenceBatchIdempotency, + ) -> Result, DispatchError> { + let key = composite_claim_lock_key(claim); + let serializer = { + let mut locks = self.composite_claim_locks.lock().map_err(|error| { + DispatchError::Internal(format!( + "composite claim serializer lock poisoned: {error}" + )) + })?; + locks.retain(|_, lock| lock.strong_count() > 0); + if let Some(lock) = locks.get(&key).and_then(std::sync::Weak::upgrade) { + lock + } else { + if locks.len() >= MAX_ACTIVE_COMPOSITE_CLAIM_LOCKS { + return Err(DispatchError::Deferred { retry_after_ms: 1 }); + } + let lock = Arc::new(tokio::sync::Mutex::new(())); + locks.insert(key, Arc::downgrade(&lock)); + lock + } + }; + Ok(serializer.lock_owned().await) + } + pub(super) fn composite_metadata_for( &self, tenant: &TenantId, @@ -138,8 +387,6 @@ impl crate::state::ServerState { validate_sub_writes(&metadata, &sub_writes)?; let parent_idempotency = composite_parent_idempotency(agent_ctx, callback_params); - let _commons_guardrail_lock = self.acquire_commons_write_guardrail_lock(tenant).await; - let composite_action_context = format!("composite:{entity_type}.{action}"); let mut composite_agent_ctx = agent_ctx.clone(); composite_agent_ctx.security_ctx = Some( @@ -150,6 +397,65 @@ impl crate::state::ServerState { .with_action_context(composite_action_context), ); + let parent = AtomicCompositeParent { + tenant, + entity_type, + entity_id, + action, + idempotency: &parent_idempotency, + record_event: metadata.record_parent_event, + agent_ctx: &composite_agent_ctx, + }; + let intent_prepared = prepare_composite_intent_sub_writes(parent, &sub_writes, &metadata); + let batch_claim = composite_batch_claim(parent, &intent_prepared)?; + // The durable append is still the final cross-backend authority, but the + // current runtime is single-process. Serialize one raw batch namespace so + // an identical callback cannot observe "claim absent", lose a + // state-dependent preflight race to its peer, and return a conflict before + // reaching the backend's exact-claim replay. + let _claim_lock = self.acquire_composite_claim_lock(&batch_claim).await?; + let event_journal = self.event_journal(); + let batch_already_committed = match event_journal.as_ref() { + Some((store, _)) => store + .batch_idempotency_committed(&batch_claim) + .await + .map_err(composite_batch_persistence_error)?, + None => false, + }; + + let _commons_guardrail_lock = self.acquire_commons_write_guardrail_lock(tenant).await; + if batch_already_committed { + if !self + .apply_composite_sub_writes_atomic(parent, &intent_prepared, batch_claim, true) + .await? + { + return Err(DispatchError::Internal( + "committed composite batch cannot be repaired without its durable event journal" + .to_string(), + )); + } + return Ok(true); + } + + let mut intended_reservation = build_composite_event( + tenant, + entity_type, + entity_id, + action, + &parent_idempotency, + &intent_prepared, + ); + intended_reservation + .intent_hash + .clone_from(&batch_claim.intent_hash); + intended_reservation + .composite_idempotency_key + .clone_from(&batch_claim.idempotency_key); + if let Some((store, _)) = event_journal.as_ref() { + self.effectful_composite_reservation_exists(store, parent, &intended_reservation) + .await?; + } + let prepared_sub_writes = self .prepare_composite_sub_writes( tenant, @@ -165,21 +471,21 @@ impl crate::state::ServerState { if self .apply_composite_sub_writes_atomic( - AtomicCompositeParent { - tenant, - entity_type, - entity_id, - action, - idempotency: &parent_idempotency, - record_event: metadata.record_parent_event, - }, + parent, &prepared_sub_writes, + batch_claim.clone(), + false, ) .await? { return Ok(true); } + if let Some((store, _)) = event_journal.as_ref() { + self.persist_effectful_composite_reservation(store, parent, &intended_reservation) + .await?; + } + for prepared in prepared_sub_writes { let mut sub_agent_ctx = composite_agent_ctx.clone(); sub_agent_ctx.idempotency_key = Some(prepared.idempotency_key); @@ -210,743 +516,6 @@ impl crate::state::ServerState { Ok(true) } - - async fn apply_composite_sub_writes_atomic( - &self, - parent: AtomicCompositeParent<'_>, - prepared_sub_writes: &[PreparedCompositeSubWrite], - ) -> Result { - let tenant = parent.tenant; - let parent_entity_type = parent.entity_type; - let parent_entity_id = parent.entity_id; - let parent_action = parent.action; - let parent_idempotency = parent.idempotency; - - let Some((store, backend)) = self.event_journal() else { - return Ok(false); - }; - if prepared_sub_writes.is_empty() { - return Ok(true); - } - - let field_sync_mode = self.composite_batch_field_sync_mode(tenant, backend); - let blob_store = self.blob_store_for_tenant(tenant).ok(); - let mut streams: BTreeMap = BTreeMap::new(); - let parent_persistence_id = format!("{tenant}:{parent_entity_type}:{parent_entity_id}"); - let timing_enabled = prepared_sub_writes.len() >= 10; - let total_started_at = timing_enabled.then(std::time::Instant::now); - let parent_started_at = timing_enabled.then(std::time::Instant::now); - - if parent.record_event - && !self - .composite_event_already_persisted( - &store, - &parent_persistence_id, - parent_idempotency, - ) - .await? - { - self.ensure_atomic_composite_stream( - &mut streams, - tenant, - parent_entity_type, - parent_entity_id, - None, - false, - ) - .await?; - let event = build_composite_event( - tenant, - parent_entity_type, - parent_entity_id, - parent_action, - parent_idempotency, - prepared_sub_writes, - ); - let stream = streams - .get_mut(&parent_persistence_id) - .expect("parent stream inserted before composite event append"); - stream - .events - .push(composite_event_envelope(&parent_persistence_id, &event)?); - stream.state.sequence_nr = stream.state.sequence_nr.saturating_add(1); - } - let parent_ms = parent_started_at.map(|started| started.elapsed().as_millis() as u64); - - let stage_started_at = timing_enabled.then(std::time::Instant::now); - for write in prepared_sub_writes { - let persistence_id = format!("{tenant}:{}:{}", write.entity_type, write.entity_id); - self.ensure_atomic_composite_stream( - &mut streams, - tenant, - &write.entity_type, - &write.entity_id, - write.preflight_target.as_ref(), - write.uses_parent_gate && write.action == "Create", - ) - .await?; - - let table = self.transition_table_for_dispatch(tenant, &write.entity_type)?; - let cross_entity_booleans = - if table_has_cross_entity_guards_for_action(&table, &write.action) { - self.resolve_cross_entity_guards( - tenant, - &write.entity_type, - &write.entity_id, - &write.action, - ) - .await - } else { - BTreeMap::new() - }; - let stream = streams - .get_mut(&persistence_id) - .expect("stream inserted before processing sub-write"); - - let incomplete_pack_object_repair = - is_incomplete_existing_pack_object_create(write, stream); - - if should_skip_existing_pack_object_create(write, stream) { - continue; - } - - if !incomplete_pack_object_repair - && stream - .state - .has_processed_idempotency_key(&write.idempotency_key) - { - continue; - } - - validate_composite_ref_compare_and_set( - parent_entity_type, - parent_action, - write, - stream, - )?; - - let result = process_action_with_xref_and_field_mode( - &mut stream.state, - &table, - &write.action, - &write.params, - &cross_entity_booleans, - field_sync_mode, - ); - if !result.success { - return Err(DispatchError::Internal(result.error.unwrap_or_else(|| { - format!( - "composite {parent_entity_type}.{parent_action} sub-write {} failed during atomic staging", - write.idx - ) - }))); - } - if !result.custom_effects.is_empty() - || !result.scheduled_actions.is_empty() - || !result.spawn_requests.is_empty() - { - return Ok(false); - } - if !result.overflow_blobs.is_empty() { - let blob_store = blob_store.as_ref().ok_or_else(|| { - DispatchError::Internal( - "field-overflow blobs require a configured object blob store".to_string(), - ) - })?; - crate::blobs::put_overflow_blobs(blob_store, &result.overflow_blobs) - .await - .map_err(|e| { - DispatchError::Internal(format!( - "field-overflow blob persistence failed during composite batch: {e}" - )) - })?; - } - - let mut event = result - .event - .expect("successful process_action returns an event"); - event.idempotency_key = Some(write.idempotency_key.clone()); - stream - .events - .push(composite_envelope(&persistence_id, &event)?); - stream.state.sequence_nr = stream.state.sequence_nr.saturating_add(1); - stream.state.push_event_bounded(event); - } - let stage_ms = stage_started_at.map(|started| started.elapsed().as_millis() as u64); - - let appends = streams - .iter() - .filter(|(_, stream)| !stream.events.is_empty()) - .map(|(persistence_id, stream)| PersistenceAppend { - persistence_id: persistence_id.clone(), - expected_sequence: stream.expected_sequence, - events: stream.events.clone(), - }) - .collect::>(); - if appends.is_empty() { - return Ok(true); - } - - let append_started_at = timing_enabled.then(std::time::Instant::now); - store - .append_batch(&appends) - .await - .map_err(composite_batch_persistence_error)?; - let append_ms = append_started_at.map(|started| started.elapsed().as_millis() as u64); - - let projection_collect_started_at = timing_enabled.then(std::time::Instant::now); - self.update_composite_query_projections(tenant, &streams) - .await?; - let projection_collect_ms = - projection_collect_started_at.map(|started| started.elapsed().as_millis() as u64); - - let projection_write_started_at = timing_enabled.then(std::time::Instant::now); - let projection_write_ms = - projection_write_started_at.map(|started| started.elapsed().as_millis() as u64); - - let reload_started_at = timing_enabled.then(std::time::Instant::now); - for stream in streams.values() { - if stream.events.is_empty() { - continue; - } - if !stream.target_existed { - continue; - } - self.stop_and_remove_entity(tenant, &stream.entity_type, &stream.entity_id); - if stream.state.status == "Deleted" { - continue; - } - if !self - .ensure_entity_loaded(tenant, &stream.entity_type, &stream.entity_id) - .await - { - return Err(DispatchError::Internal(format!( - "composite batch committed {}:{} but failed to reload it", - stream.entity_type, stream.entity_id - ))); - } - } - let reload_ms = reload_started_at.map(|started| started.elapsed().as_millis() as u64); - if let Some(started) = total_started_at { - tracing::info!( - tenant = %tenant, - parent_entity_type, - parent_entity_id, - parent_action, - sub_writes = prepared_sub_writes.len(), - streams = streams.len(), - parent_ms = parent_ms.unwrap_or_default(), - stage_ms = stage_ms.unwrap_or_default(), - append_ms = append_ms.unwrap_or_default(), - projection_collect_ms = projection_collect_ms.unwrap_or_default(), - projection_write_ms = projection_write_ms.unwrap_or_default(), - reload_ms = reload_ms.unwrap_or_default(), - total_ms = started.elapsed().as_millis() as u64, - "composite atomic batch timing" - ); - } - - Ok(true) - } - - async fn ensure_atomic_composite_stream( - &self, - streams: &mut BTreeMap, - tenant: &TenantId, - entity_type: &str, - entity_id: &str, - preflight_target: Option<&PreflightCompositeTarget>, - suppress_bootstrap_event: bool, - ) -> Result<(), DispatchError> { - let persistence_id = format!("{tenant}:{entity_type}:{entity_id}"); - if streams.contains_key(&persistence_id) { - return Ok(()); - } - - let (target_exists, mut state) = if let Some(target) = preflight_target { - (target.target_existed, target.state.clone()) - } else { - let table = self.transition_table_for_dispatch(tenant, entity_type)?; - let target_exists = self - .ensure_entity_loaded(tenant, entity_type, entity_id) - .await; - let state = if target_exists { - self.get_tenant_entity_state(tenant, entity_type, entity_id) - .await - .map_err(DispatchError::Internal)? - .state - } else { - synthetic_initial_state(entity_type, entity_id, &table) - }; - (target_exists, state) - }; - let expected_sequence = state.sequence_nr; - let mut events = Vec::new(); - if !suppress_bootstrap_event - && !target_exists - && expected_sequence == 0 - && state.total_event_count == 0 - { - let bootstrap = crate::entity_actor::EntityEvent { - action: "Created".to_string(), - from_status: String::new(), - to_status: state.status.clone(), - timestamp: sim_now(), - params: serde_json::json!({}), - idempotency_key: None, - }; - events.push(composite_envelope(&persistence_id, &bootstrap)?); - state.sequence_nr = state.sequence_nr.saturating_add(1); - state.push_event_bounded(bootstrap); - } - streams.insert( - persistence_id, - AtomicCompositeStream { - entity_type: entity_type.to_string(), - entity_id: entity_id.to_string(), - target_existed: target_exists, - state, - expected_sequence, - events, - }, - ); - Ok(()) - } - - async fn composite_event_already_persisted( - &self, - store: &crate::storage::BoxedEventStore, - parent_persistence_id: &str, - parent_idempotency: &str, - ) -> Result { - let envelopes = store - .read_events(parent_persistence_id, 0) - .await - .map_err(|e| { - DispatchError::Internal(format!( - "failed to read parent journal before CompositeEvent append: {e}" - )) - })?; - Ok(envelopes.iter().any(|env| { - env.event_type == COMPOSITE_EVENT_TYPE - && serde_json::from_value::(env.payload.clone()) - .is_ok_and(|event| event.composite_idempotency_key == parent_idempotency) - })) - } - - fn composite_batch_field_sync_mode( - &self, - tenant: &TenantId, - backend: BackendLabel, - ) -> FieldSyncMode { - match backend { - BackendLabel::Turso | BackendLabel::TursoRouted => FieldSyncMode::blob_refs_default(), - _ if self.blob_store_for_tenant(tenant).is_ok() => FieldSyncMode::blob_refs_default(), - _ => FieldSyncMode::InlineTruncate, - } - } - - #[allow(clippy::too_many_arguments)] - async fn prepare_composite_sub_writes( - &self, - tenant: &TenantId, - entity_type: &str, - entity_id: &str, - action: &str, - sub_writes: &[CompositeSubWrite], - metadata: &CompositeActionMetadata, - composite_agent_ctx: &AgentContext, - parent_idempotency: &str, - ) -> Result, DispatchError> { - let sub_security_ctx = composite_agent_ctx.security_ctx.as_ref().ok_or_else(|| { - DispatchError::Internal( - "composite sub-write authorization requires a security context".to_string(), - ) - })?; - let mut prepared = Vec::with_capacity(sub_writes.len()); - let mut governed_cache = BTreeMap::new(); - let mut create_auth_defaults_cache = BTreeMap::new(); - - for (idx, sub_write) in sub_writes.iter().cloned().enumerate() { - let sub_entity_type = sub_write.entity_type.clone(); - let sub_entity_id = sub_write.entity_id.clone(); - let sub_action = sub_write.action.clone(); - let sub_params = normalize_sub_write_params(sub_write); - - let governed = match governed_cache.get(&sub_entity_type) { - Some(governed) => *governed, - None => { - let governed = self - .is_entity_type_governed(tenant, &sub_entity_type) - .map_err(DispatchError::Internal)?; - governed_cache.insert(sub_entity_type.clone(), governed); - governed - } - }; - if !governed { - return Err(DispatchError::Ungoverned(sub_entity_type)); - } - - let use_parent_gate = - composite_sub_write_uses_parent_gate(metadata, &sub_entity_type, &sub_action); - let resource_attrs = if use_parent_gate { - None - } else if sub_action == "Create" { - if !create_auth_defaults_cache.contains_key(&sub_entity_type) { - create_auth_defaults_cache.insert( - sub_entity_type.clone(), - self.composite_create_auth_defaults(tenant, &sub_entity_type)?, - ); - } - let defaults = create_auth_defaults_cache - .get(&sub_entity_type) - .expect("create auth defaults inserted before use"); - Some(composite_create_resource_attrs_from_defaults( - &sub_entity_id, - &sub_params, - defaults, - )) - } else { - Some( - self.composite_sub_write_auth_resource_attrs( - tenant, - &sub_entity_type, - &sub_entity_id, - &sub_action, - &sub_params, - ) - .await?, - ) - }; - - if let Some(resource_attrs) = resource_attrs { - self.authorize_with_context( - sub_security_ctx, - &sub_action, - &sub_entity_type, - &resource_attrs, - tenant.as_str(), - ) - .map_err(|denial| { - DispatchError::AuthzDenied(format!( - "composite {entity_type}.{action} sub-write {idx} denied for {sub_entity_type}.{sub_action}: {denial}" - )) - })?; - } - - prepared.push(PreparedCompositeSubWrite { - idx, - entity_type: sub_entity_type, - entity_id: sub_entity_id, - action: sub_action, - params: sub_params, - idempotency_key: format!( - "composite:{tenant}:{entity_type}:{entity_id}:{action}:{parent_idempotency}:subwrite:{idx}" - ), - preflight_target: None, - uses_parent_gate: use_parent_gate, - }); - } - - let known_absent_create_targets = self - .composite_known_absent_create_targets(tenant, &prepared) - .await?; - - for write in &mut prepared { - let known_absent_create = known_absent_create_targets - .contains(&(write.entity_type.clone(), write.entity_id.clone())); - write.preflight_target = Some( - self.preflight_composite_sub_write_transition( - tenant, - entity_type, - action, - write, - known_absent_create, - ) - .await?, - ); - } - - let storage_writes = prepared - .iter() - .map(|write| CommonsStorageWrite { - entity_type: write.entity_type.clone(), - entity_id: write.entity_id.clone(), - action: write.action.clone(), - fields: write.params.clone(), - }) - .collect::>(); - for write in &storage_writes { - self.enforce_commons_verified_owner_for_write( - tenant, - &write.entity_type, - &write.fields, - ) - .await - .map_err(composite_account_verification_error)?; - self.enforce_commons_app_name_unique_for_write( - tenant, - &write.entity_type, - &write.entity_id, - &write.fields, - ) - .await - .map_err(composite_app_uniqueness_error)?; - } - self.enforce_commons_storage_caps_for_writes(tenant, &storage_writes) - .await - .map_err(composite_storage_cap_error)?; - - Ok(prepared) - } - - async fn composite_known_absent_create_targets( - &self, - tenant: &TenantId, - prepared: &[PreparedCompositeSubWrite], - ) -> Result, DispatchError> { - let Some(query_plane) = self.query_plane_store() else { - return Ok(BTreeSet::new()); - }; - - let mut by_type: BTreeMap> = BTreeMap::new(); - for write in prepared { - if write.uses_parent_gate - || write.action != "Create" - || self.entity_exists(tenant, &write.entity_type, &write.entity_id) - { - continue; - } - by_type - .entry(write.entity_type.clone()) - .or_default() - .insert(write.entity_id.clone()); - } - - let mut absent = BTreeSet::new(); - for (entity_type, ids) in by_type { - let entity_ids = ids.into_iter().collect::>(); - let Some(rows) = query_plane - .load_entity_catalog_rows(tenant.as_str(), &entity_type, &entity_ids) - .await - .map_err(|e| { - DispatchError::Internal(format!( - "query projection preflight failed for composite {entity_type} creates: {e}" - )) - })? - else { - continue; - }; - - let present = rows - .into_iter() - .map(|row| row.entity_id) - .collect::>(); - for entity_id in entity_ids { - if !present.contains(&entity_id) { - absent.insert((entity_type.clone(), entity_id)); - } - } - } - - Ok(absent) - } - - async fn preflight_composite_sub_write_transition( - &self, - tenant: &TenantId, - parent_entity_type: &str, - parent_action: &str, - write: &PreparedCompositeSubWrite, - known_absent_create: bool, - ) -> Result { - let table = self.transition_table_for_dispatch(tenant, &write.entity_type)?; - let known_absent_create = known_absent_create - && write.action == "Create" - && !self.entity_exists(tenant, &write.entity_type, &write.entity_id); - let target_exists = if known_absent_create { - false - } else { - self.ensure_entity_loaded(tenant, &write.entity_type, &write.entity_id) - .await - }; - let target_state = if known_absent_create { - synthetic_initial_state(&write.entity_type, &write.entity_id, &table) - } else if target_exists { - self.get_tenant_entity_state(tenant, &write.entity_type, &write.entity_id) - .await - .map_err(DispatchError::Internal)? - .state - } else { - synthetic_initial_state(&write.entity_type, &write.entity_id, &table) - }; - - let preflight_target = PreflightCompositeTarget { - target_existed: target_exists, - state: target_state.clone(), - }; - - if target_state.has_processed_idempotency_key(&write.idempotency_key) { - return Ok(preflight_target); - } - - validate_composite_ref_preflight_compare_and_set( - parent_entity_type, - parent_action, - write, - &preflight_target, - )?; - - if !target_state.can_accept_event() { - return Err(DispatchError::Internal(format!( - "composite {parent_entity_type}.{parent_action} sub-write {} would exceed the event budget for {}:{}", - write.idx, write.entity_type, write.entity_id - ))); - } - - let cross_entity_booleans = - if table_has_cross_entity_guards_for_action(&table, &write.action) { - self.resolve_cross_entity_guards( - tenant, - &write.entity_type, - &write.entity_id, - &write.action, - ) - .await - } else { - BTreeMap::new() - }; - let eval_ctx = build_eval_context_with_xref(&target_state, &cross_entity_booleans); - match table.evaluate_ctx(&target_state.status, &eval_ctx, &write.action) { - Some(result) if result.success => Ok(preflight_target), - Some(_) => Err(DispatchError::Conflict(format!( - "composite {parent_entity_type}.{parent_action} sub-write {} would fail: action '{}' is not valid from state '{}'", - write.idx, write.action, target_state.status - ))), - None => Err(DispatchError::Internal(format!( - "composite {parent_entity_type}.{parent_action} sub-write {} would fail: unknown action '{}'", - write.idx, write.action - ))), - } - } - - async fn composite_sub_write_auth_resource_attrs( - &self, - tenant: &TenantId, - entity_type: &str, - entity_id: &str, - action: &str, - params: &Value, - ) -> Result, DispatchError> { - if action == "Create" { - return self.composite_create_resource_attrs(tenant, entity_type, entity_id, params); - } - - if !self - .ensure_entity_loaded(tenant, entity_type, entity_id) - .await - { - return Err(DispatchError::Internal(format!( - "composite sub-write target {entity_type}:{entity_id} does not exist" - ))); - } - - self.load_authz_resource_snapshot(tenant, entity_type, entity_id) - .await - .map(|snapshot| snapshot.resource_attrs) - .map_err(DispatchError::Internal) - } - - fn composite_create_auth_defaults( - &self, - tenant: &TenantId, - entity_type: &str, - ) -> Result { - let table = self.transition_table_for_dispatch(tenant, entity_type)?; - let has_spec = self - .has_registered_spec(tenant, entity_type) - .map_err(DispatchError::Internal)?; - Ok(CompositeCreateAuthDefaults { - initial_state: table.initial_state.clone(), - has_spec, - }) - } - - fn composite_create_resource_attrs( - &self, - tenant: &TenantId, - entity_type: &str, - entity_id: &str, - params: &Value, - ) -> Result, DispatchError> { - let table = self.transition_table_for_dispatch(tenant, entity_type)?; - let mut resource_attrs = BTreeMap::new(); - resource_attrs.insert("id".to_string(), Value::String(entity_id.to_string())); - resource_attrs.insert( - "status".to_string(), - Value::String(table.initial_state.clone()), - ); - if let Value::Object(fields) = params { - for (key, value) in fields { - resource_attrs.insert(key.clone(), value.clone()); - } - } - let has_spec = self - .has_registered_spec(tenant, entity_type) - .map_err(DispatchError::Internal)?; - resource_attrs.insert("has_spec".to_string(), Value::Bool(has_spec)); - Ok(resource_attrs) - } - - fn transition_table_for_dispatch( - &self, - tenant: &TenantId, - entity_type: &str, - ) -> Result, DispatchError> { - if let Some(table) = self - .registry - .read() - .map_err(|e| DispatchError::Internal(format!("registry lock poisoned: {e}")))? - .get_table(tenant, entity_type) - { - return Ok(table); - } - - self.transition_tables - .get(entity_type) - .cloned() - .ok_or_else(|| DispatchError::Ungoverned(entity_type.to_string())) - } - - #[allow(dead_code)] - async fn ensure_composite_entry_transition_allowed( - &self, - tenant: &TenantId, - entity_type: &str, - entity_id: &str, - action: &str, - ) -> Result<(), DispatchError> { - let table = self.transition_table_for_dispatch(tenant, entity_type)?; - let current = self - .get_tenant_entity_state(tenant, entity_type, entity_id) - .await - .map_err(DispatchError::Internal)?; - let cross_entity_booleans = self - .resolve_cross_entity_guards(tenant, entity_type, entity_id, action) - .await; - let eval_ctx = build_eval_context_with_xref(¤t.state, &cross_entity_booleans); - - match table.evaluate_ctx(¤t.state.status, &eval_ctx, action) { - Some(result) if result.success => Ok(()), - Some(_) => Err(DispatchError::Internal(format!( - "Composite action '{action}' not valid from state '{}'", - current.state.status - ))), - None => Err(DispatchError::Internal(format!( - "Unknown composite action: {action}" - ))), - } - } } #[cfg(test)] diff --git a/crates/temper-server/src/state/dispatch/composite/atomic.rs b/crates/temper-server/src/state/dispatch/composite/atomic.rs new file mode 100644 index 000000000..9db56ca85 --- /dev/null +++ b/crates/temper-server/src/state/dispatch/composite/atomic.rs @@ -0,0 +1,491 @@ +use super::*; + +impl crate::state::ServerState { + pub(super) async fn apply_composite_sub_writes_atomic( + &self, + parent: AtomicCompositeParent<'_>, + prepared_sub_writes: &[PreparedCompositeSubWrite], + batch_claim: PersistenceBatchIdempotency, + batch_already_committed: bool, + ) -> Result { + let tenant = parent.tenant; + let parent_entity_type = parent.entity_type; + let parent_entity_id = parent.entity_id; + let parent_action = parent.action; + let parent_idempotency = parent.idempotency; + + let Some((store, backend)) = self.event_journal() else { + return Ok(false); + }; + if prepared_sub_writes.is_empty() { + return Ok(true); + } + + // Capture one coherent table per participating entity type before any + // recovery or staging await point. Every event, key row, and activation + // token in this atomic attempt is derived from that same table. + let mut captured_tables = BTreeMap::new(); + if parent.record_event { + captured_tables.insert( + parent_entity_type.to_string(), + self.transition_table_for_dispatch(tenant, parent_entity_type)?, + ); + } + for write in prepared_sub_writes { + if !captured_tables.contains_key(&write.entity_type) { + captured_tables.insert( + write.entity_type.clone(), + self.transition_table_for_dispatch(tenant, &write.entity_type)?, + ); + } + } + + let field_sync_mode = self.composite_batch_field_sync_mode(tenant, backend); + let blob_store = self.blob_store_for_tenant(tenant).ok(); + let mut streams: BTreeMap = BTreeMap::new(); + let parent_persistence_id = format!("{tenant}:{parent_entity_type}:{parent_entity_id}"); + let mut composite_event = build_composite_event( + tenant, + parent_entity_type, + parent_entity_id, + parent_action, + parent_idempotency, + prepared_sub_writes, + ); + composite_event + .intent_hash + .clone_from(&batch_claim.intent_hash); + composite_event + .composite_idempotency_key + .clone_from(&batch_claim.idempotency_key); + let timing_enabled = prepared_sub_writes.len() >= 10; + let total_started_at = timing_enabled.then(std::time::Instant::now); // determinism-ok: observability only + let parent_started_at = timing_enabled.then(std::time::Instant::now); // determinism-ok: observability only + + if parent.record_event { + self.ensure_atomic_composite_stream( + &mut streams, + &store, + backend, + tenant, + parent_entity_type, + parent_entity_id, + captured_tables + .get(parent_entity_type) + .expect("parent table captured before stream recovery"), + None, + false, + ) + .await?; + let stream = streams + .get_mut(&parent_persistence_id) + .expect("parent stream inserted before composite event append"); + if !batch_already_committed { + let parent_audit_already_persisted = stream + .state + .has_processed_idempotency_key(parent_idempotency); + if !parent_audit_already_persisted && !stream.state.can_accept_event() { + return Err(DispatchError::Internal(format!( + "composite {parent_entity_type}.{parent_action} parent audit would exceed the event budget for {parent_entity_type}:{parent_entity_id}" + ))); + } + if !parent_audit_already_persisted { + stream.events.push(composite_event_envelope( + &parent_persistence_id, + &composite_event, + )?); + stream.state.sequence_nr = stream.state.sequence_nr.saturating_add(1); + stream.state.record_internal_envelope(); + stream.state.record_durable_idempotency_key( + parent_idempotency, + stream.state.sequence_nr, + ); + } + } + } + let parent_ms = parent_started_at.map(|started| started.elapsed().as_millis() as u64); + + let stage_started_at = timing_enabled.then(std::time::Instant::now); // determinism-ok: observability only + let mut post_commit = Vec::with_capacity(prepared_sub_writes.len()); + for write in prepared_sub_writes { + let persistence_id = format!("{tenant}:{}:{}", write.entity_type, write.entity_id); + self.ensure_atomic_composite_stream( + &mut streams, + &store, + backend, + tenant, + &write.entity_type, + &write.entity_id, + captured_tables + .get(&write.entity_type) + .expect("sub-write table captured before stream recovery"), + write.preflight_target.as_ref(), + write.uses_parent_gate && write.action == "Create", + ) + .await?; + + if batch_already_committed { + continue; + } + + let table = streams + .get(&persistence_id) + .expect("stream inserted before table lookup") + .table + .clone(); + let cross_entity_booleans = + if table_has_cross_entity_guards_for_action(&table, &write.action) { + self.resolve_cross_entity_guards( + tenant, + &write.entity_type, + &write.entity_id, + &write.action, + ) + .await + } else { + BTreeMap::new() + }; + let stream = streams + .get_mut(&persistence_id) + .expect("stream inserted before processing sub-write"); + + let incomplete_pack_object_repair = + is_incomplete_existing_pack_object_create(write, stream); + + if should_skip_existing_pack_object_create(write, stream) { + continue; + } + + if !incomplete_pack_object_repair + && stream + .state + .has_processed_idempotency_key(&write.idempotency_key) + { + continue; + } + + validate_composite_ref_compare_and_set( + parent_entity_type, + parent_action, + write, + stream, + )?; + + let result = process_action_with_xref_and_field_mode( + &mut stream.state, + &table, + &write.action, + &write.params, + &cross_entity_booleans, + field_sync_mode, + ); + if !result.success { + return Err(DispatchError::Internal(result.error.unwrap_or_else(|| { + format!( + "composite {parent_entity_type}.{parent_action} sub-write {} failed during atomic staging", + write.idx + ) + }))); + } + if !result.custom_effects.is_empty() + || !result.scheduled_actions.is_empty() + || !result.spawn_requests.is_empty() + { + // These obligations are owned by actor idempotency and are not + // represented in the atomic batch claim. Decline before the + // append so an ambiguous successful commit cannot make their + // first execution indistinguishable from a completed replay. + return Ok(false); + } + if !result.overflow_blobs.is_empty() { + let blob_store = blob_store.as_ref().ok_or_else(|| { + DispatchError::Internal( + "field-overflow blobs require a configured object blob store".to_string(), + ) + })?; + crate::blobs::put_overflow_blobs(blob_store, &result.overflow_blobs) + .await + .map_err(|e| { + DispatchError::Internal(format!( + "field-overflow blob persistence failed during composite batch: {e}" + )) + })?; + } + + let mut event = result + .event + .expect("successful process_action returns an event"); + event.idempotency_key = Some(write.idempotency_key.clone()); + stream + .events + .push(composite_envelope(&persistence_id, &event)?); + stream.state.sequence_nr = stream.state.sequence_nr.saturating_add(1); + stream.state.push_event_bounded(event); + post_commit.push(AtomicCompositePostCommit { + entity_type: write.entity_type.clone(), + entity_id: write.entity_id.clone(), + action: write.action.clone(), + params: write.params.clone(), + idempotency_key: write.idempotency_key.clone(), + response: EntityResponse { + success: true, + state: stream.state.clone(), + error: None, + custom_effects: result.custom_effects, + scheduled_actions: result.scheduled_actions, + spawn_requests: result.spawn_requests, + spec_governed: true, + }, + }); + } + let stage_ms = stage_started_at.map(|started| started.elapsed().as_millis() as u64); + + let mut appends = Vec::with_capacity(streams.len()); + for (persistence_id, stream) in &mut streams { + let mut events = Vec::with_capacity( + stream.events.len() + usize::from(stream.materialization_baseline.is_some()), + ); + if let Some(baseline) = stream.materialization_baseline.take() { + events.push( + state_materialization_envelope(persistence_id, &baseline, sim_now()) + .map_err(composite_batch_persistence_error)?, + ); + stream.state.sequence_nr = stream.state.sequence_nr.saturating_add(1); + } + events.extend(stream.events.iter().cloned()); + // Empty is an exact declared-key set and must delete ownership + // inherited from an older, keyed contract. + let reconcile_keys = true; + let key_set_signature = crate::key_index::declared_key_write_contract(&stream.table); + let key_rows = crate::key_index::derive_entity_key_rows( + &stream.table.keys, + &stream.state.fields, + stream.state.status != "Deleted", + ); + appends.push(PersistenceAppend { + persistence_id: persistence_id.clone(), + expected_sequence: stream.expected_sequence, + events, + key_rows, + reconcile_keys, + key_set_signature: Some(key_set_signature), + snapshot_source: stream.snapshot_source.clone(), + batch_idempotency: None, + }); + } + debug_assert!( + !appends.is_empty(), + "a non-empty composite must capture at least one target stream" + ); + appends[0].batch_idempotency = Some(batch_claim); + + let append_started_at = timing_enabled.then(std::time::Instant::now); // determinism-ok: observability only + let append_results = store + .append_batch(&appends) + .await + .map_err(composite_batch_persistence_error)?; + let append_ms = append_started_at.map(|started| started.elapsed().as_millis() as u64); + let batch_replayed = append_results + .iter() + .any(|result| result.batch_already_applied); + if batch_already_committed && !batch_replayed { + return Err(DispatchError::Internal( + "durable composite claim disappeared between retry probe and atomic replay" + .to_string(), + )); + } + + if !batch_replayed { + for effect in post_commit { + let mut sub_agent_ctx = parent.agent_ctx.clone(); + sub_agent_ctx.idempotency_key = Some(effect.idempotency_key.clone()); + let context = PostDispatchContext { + tenant, + entity_type: &effect.entity_type, + entity_id: &effect.entity_id, + action: &effect.action, + agent_ctx: &sub_agent_ctx, + dispatch_idempotency_key: Some(&effect.idempotency_key), + action_params: &effect.params, + await_integration: false, + }; + self.run_post_dispatch_effects(&context, effect.response) + .await; + } + } + + let projection_collect_started_at = timing_enabled.then(std::time::Instant::now); // determinism-ok: observability only + self.update_composite_query_projections(tenant, &streams) + .await?; + let projection_collect_ms = + projection_collect_started_at.map(|started| started.elapsed().as_millis() as u64); + + let projection_write_started_at = timing_enabled.then(std::time::Instant::now); // determinism-ok: observability only + let projection_write_ms = + projection_write_started_at.map(|started| started.elapsed().as_millis() as u64); + + let reload_started_at = timing_enabled.then(std::time::Instant::now); // determinism-ok: observability only + for stream in streams.values() { + if !stream.target_existed { + continue; + } + self.stop_and_remove_entity(tenant, &stream.entity_type, &stream.entity_id); + if stream.state.status == "Deleted" { + continue; + } + if !self + .ensure_entity_loaded(tenant, &stream.entity_type, &stream.entity_id) + .await + { + return Err(DispatchError::Internal(format!( + "composite batch committed {}:{} but failed to reload it", + stream.entity_type, stream.entity_id + ))); + } + } + let reload_ms = reload_started_at.map(|started| started.elapsed().as_millis() as u64); + if let Some(started) = total_started_at { + tracing::info!( + tenant = %tenant, + parent_entity_type, + parent_entity_id, + parent_action, + sub_writes = prepared_sub_writes.len(), + streams = streams.len(), + parent_ms = parent_ms.unwrap_or_default(), + stage_ms = stage_ms.unwrap_or_default(), + append_ms = append_ms.unwrap_or_default(), + projection_collect_ms = projection_collect_ms.unwrap_or_default(), + projection_write_ms = projection_write_ms.unwrap_or_default(), + reload_ms = reload_ms.unwrap_or_default(), + batch_replayed, + total_ms = started.elapsed().as_millis() as u64, + "composite atomic batch timing" + ); + } + + Ok(true) + } + + #[expect( + clippy::too_many_arguments, + reason = "composite stream capture is an explicit multi-entity transaction boundary" + )] + async fn ensure_atomic_composite_stream( + &self, + streams: &mut BTreeMap, + store: &crate::storage::BoxedEventStore, + backend: BackendLabel, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + table: &TransitionTable, + preflight_target: Option<&PreflightCompositeTarget>, + suppress_bootstrap_event: bool, + ) -> Result<(), DispatchError> { + let persistence_id = format!("{tenant}:{entity_type}:{entity_id}"); + if streams.contains_key(&persistence_id) { + return Ok(()); + } + + let blob_store = self.blob_store_for_tenant(tenant).ok(); + let initial_fields = serde_json::json!({}); + let source = recover_entity_state_from_stable_sources(EntityRecoveryContext { + tenant: tenant.as_str(), + entity_type, + entity_id, + table, + store, + backend, + initial_fields: &initial_fields, + blob_store: blob_store.as_ref(), + }) + .await + .map_err(|error| { + DispatchError::Internal(format!( + "failed to capture durable composite source for {entity_type}:{entity_id}: {error}" + )) + })?; + let expected_sequence = source.journal_sequence; + let has_snapshot = source.snapshot.is_some(); + let snapshot_source = match source.snapshot { + Some(snapshot) => SnapshotSourceFence::Exact { + sequence_nr: snapshot.sequence_nr, + state: snapshot.state, + }, + None => SnapshotSourceFence::Absent, + }; + let (target_exists, mut state) = if let Some(state) = source.state { + (true, state) + } else { + ( + false, + synthetic_initial_state(entity_type, entity_id, table), + ) + }; + // Composite optimistic concurrency, like actor appends, is owned by the + // journal high-water rather than a snapshot-only aggregate sequence. + state.sequence_nr = expected_sequence; + if target_exists && expected_sequence == 0 && has_snapshot { + state.last_snapshot_sequence_nr = 0; + state.events_since_snapshot = 0; + } + if let Some(preflight_target) = preflight_target { + let preflight_changed = preflight_target.target_existed != target_exists + || (target_exists + && (preflight_target.state.sequence_nr != state.sequence_nr + || preflight_target.state.status != state.status + || preflight_target.state.fields != state.fields + || preflight_target.state.item_count != state.item_count + || preflight_target.state.counters != state.counters + || preflight_target.state.booleans != state.booleans + || preflight_target.state.lists != state.lists + || preflight_target.state.processed_idempotency_keys + != state.processed_idempotency_keys + || preflight_target.state.can_accept_event() != state.can_accept_event())); + if preflight_changed { + return Err(DispatchError::Conflict(format!( + "composite target {entity_type}:{entity_id} changed after authorization preflight; retry the composite action" + ))); + } + } + let materialization_baseline = + (target_exists && expected_sequence == 0 && has_snapshot).then(|| state.clone()); + if materialization_baseline.is_some() { + state.record_internal_envelope(); + } + let mut events = Vec::new(); + if !suppress_bootstrap_event + && !target_exists + && expected_sequence == 0 + && state.total_event_count == 0 + { + let bootstrap = crate::entity_actor::EntityEvent { + action: "Created".to_string(), + from_status: String::new(), + to_status: state.status.clone(), + timestamp: sim_now(), + params: serde_json::json!({}), + idempotency_key: None, + }; + events.push(composite_envelope(&persistence_id, &bootstrap)?); + state.sequence_nr = state.sequence_nr.saturating_add(1); + state.push_event_bounded(bootstrap); + } + streams.insert( + persistence_id, + AtomicCompositeStream { + entity_type: entity_type.to_string(), + entity_id: entity_id.to_string(), + table: table.clone(), + target_existed: target_exists, + state, + expected_sequence, + snapshot_source, + materialization_baseline, + events, + }, + ); + Ok(()) + } +} diff --git a/crates/temper-server/src/state/dispatch/composite/helpers.rs b/crates/temper-server/src/state/dispatch/composite/helpers.rs index 99ec9525f..7d02e7dde 100644 --- a/crates/temper-server/src/state/dispatch/composite/helpers.rs +++ b/crates/temper-server/src/state/dispatch/composite/helpers.rs @@ -1,5 +1,19 @@ use super::*; +impl crate::state::ServerState { + pub(super) fn composite_batch_field_sync_mode( + &self, + tenant: &TenantId, + backend: BackendLabel, + ) -> FieldSyncMode { + match backend { + BackendLabel::Turso | BackendLabel::TursoRouted => FieldSyncMode::blob_refs_default(), + _ if self.blob_store_for_tenant(tenant).is_ok() => FieldSyncMode::blob_refs_default(), + _ => FieldSyncMode::InlineTruncate, + } + } +} + pub(super) fn synthetic_initial_state( entity_type: &str, entity_id: &str, @@ -289,6 +303,21 @@ pub(super) fn composite_parent_idempotency( format!("implicit:{:x}", hasher.finalize()) } +pub(super) fn canonical_json(value: &Value) -> Value { + match value { + Value::Object(fields) => Value::Object( + fields + .iter() + .map(|(key, value)| (key.clone(), canonical_json(value))) + .collect::>() + .into_iter() + .collect(), + ), + Value::Array(values) => Value::Array(values.iter().map(canonical_json).collect()), + scalar => scalar.clone(), + } +} + pub(super) fn build_composite_event( tenant: &TenantId, parent_entity_type: &str, @@ -303,6 +332,7 @@ pub(super) fn build_composite_event( parent_entity_id: parent_entity_id.to_string(), parent_action: parent_action.to_string(), composite_idempotency_key: parent_idempotency.to_string(), + intent_hash: String::new(), sub_writes: prepared_sub_writes .iter() .map(|write| CompositeEventSubWrite { @@ -359,7 +389,14 @@ pub(super) fn composite_envelope( pub(super) fn composite_batch_persistence_error(error: PersistenceError) -> DispatchError { match error { - PersistenceError::ConcurrencyViolation { .. } => { + PersistenceError::ConcurrencyViolation { .. } + | PersistenceError::KeyContractChanged { .. } + | PersistenceError::KeyContractNotActive { .. } + | PersistenceError::KeyContractActivationStale { .. } + | PersistenceError::KeyContractActivationNotReady { .. } + | PersistenceError::JournalBoundaryChanged { .. } + | PersistenceError::EntityLivenessChanged { .. } + | PersistenceError::SnapshotGenerationChanged => { DispatchError::Conflict(format!("composite batch persistence conflict: {error}")) } PersistenceError::Serialization(_) | PersistenceError::Storage(_) => { diff --git a/crates/temper-server/src/state/dispatch/composite/preflight.rs b/crates/temper-server/src/state/dispatch/composite/preflight.rs new file mode 100644 index 000000000..8782e4849 --- /dev/null +++ b/crates/temper-server/src/state/dispatch/composite/preflight.rs @@ -0,0 +1,408 @@ +use super::*; + +impl crate::state::ServerState { + #[expect( + clippy::too_many_arguments, + reason = "preflight binds one complete authorized composite operation" + )] + pub(super) async fn prepare_composite_sub_writes( + &self, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + action: &str, + sub_writes: &[CompositeSubWrite], + metadata: &CompositeActionMetadata, + composite_agent_ctx: &AgentContext, + parent_idempotency: &str, + ) -> Result, DispatchError> { + let sub_security_ctx = composite_agent_ctx.security_ctx.as_ref().ok_or_else(|| { + DispatchError::Internal( + "composite sub-write authorization requires a security context".to_string(), + ) + })?; + let mut prepared = Vec::with_capacity(sub_writes.len()); + let mut governed_cache = BTreeMap::new(); + let mut create_auth_defaults_cache = BTreeMap::new(); + + for (idx, sub_write) in sub_writes.iter().cloned().enumerate() { + let sub_entity_type = sub_write.entity_type.clone(); + let sub_entity_id = sub_write.entity_id.clone(); + let sub_action = sub_write.action.clone(); + let sub_params = normalize_sub_write_params(sub_write); + + let governed = match governed_cache.get(&sub_entity_type) { + Some(governed) => *governed, + None => { + let governed = self + .is_entity_type_governed(tenant, &sub_entity_type) + .map_err(DispatchError::Internal)?; + governed_cache.insert(sub_entity_type.clone(), governed); + governed + } + }; + if !governed { + return Err(DispatchError::Ungoverned(sub_entity_type)); + } + + let use_parent_gate = + composite_sub_write_uses_parent_gate(metadata, &sub_entity_type, &sub_action); + let resource_attrs = if use_parent_gate { + None + } else if sub_action == "Create" { + if !create_auth_defaults_cache.contains_key(&sub_entity_type) { + create_auth_defaults_cache.insert( + sub_entity_type.clone(), + self.composite_create_auth_defaults(tenant, &sub_entity_type)?, + ); + } + let defaults = create_auth_defaults_cache + .get(&sub_entity_type) + .expect("create auth defaults inserted before use"); + Some(composite_create_resource_attrs_from_defaults( + &sub_entity_id, + &sub_params, + defaults, + )) + } else { + Some( + self.composite_sub_write_auth_resource_attrs( + tenant, + &sub_entity_type, + &sub_entity_id, + &sub_action, + &sub_params, + ) + .await?, + ) + }; + + if let Some(resource_attrs) = resource_attrs { + self.authorize_with_context( + sub_security_ctx, + &sub_action, + &sub_entity_type, + &resource_attrs, + tenant.as_str(), + ) + .map_err(|denial| { + DispatchError::AuthzDenied(format!( + "composite {entity_type}.{action} sub-write {idx} denied for {sub_entity_type}.{sub_action}: {denial}" + )) + })?; + } + + prepared.push(PreparedCompositeSubWrite { + idx, + entity_type: sub_entity_type, + entity_id: sub_entity_id, + action: sub_action, + params: sub_params, + idempotency_key: format!( + "composite:{tenant}:{entity_type}:{entity_id}:{action}:{parent_idempotency}:subwrite:{idx}" + ), + preflight_target: None, + uses_parent_gate: use_parent_gate, + }); + } + + let known_absent_create_targets = self + .composite_known_absent_create_targets(tenant, &prepared) + .await?; + + for write in &mut prepared { + let known_absent_create = known_absent_create_targets + .contains(&(write.entity_type.clone(), write.entity_id.clone())); + write.preflight_target = Some( + self.preflight_composite_sub_write_transition( + tenant, + entity_type, + action, + write, + known_absent_create, + ) + .await?, + ); + } + + let storage_writes = prepared + .iter() + .map(|write| CommonsStorageWrite { + entity_type: write.entity_type.clone(), + entity_id: write.entity_id.clone(), + action: write.action.clone(), + fields: write.params.clone(), + }) + .collect::>(); + for write in &storage_writes { + self.enforce_commons_verified_owner_for_write( + tenant, + &write.entity_type, + &write.fields, + ) + .await + .map_err(composite_account_verification_error)?; + self.enforce_commons_app_name_unique_for_write( + tenant, + &write.entity_type, + &write.entity_id, + &write.fields, + ) + .await + .map_err(composite_app_uniqueness_error)?; + } + self.enforce_commons_storage_caps_for_writes(tenant, &storage_writes) + .await + .map_err(composite_storage_cap_error)?; + + Ok(prepared) + } + + async fn composite_known_absent_create_targets( + &self, + tenant: &TenantId, + prepared: &[PreparedCompositeSubWrite], + ) -> Result, DispatchError> { + let Some(query_plane) = self.query_plane_store() else { + return Ok(BTreeSet::new()); + }; + + let mut by_type: BTreeMap> = BTreeMap::new(); + for write in prepared { + if write.uses_parent_gate + || write.action != "Create" + || self.entity_exists(tenant, &write.entity_type, &write.entity_id) + { + continue; + } + by_type + .entry(write.entity_type.clone()) + .or_default() + .insert(write.entity_id.clone()); + } + + let mut absent = BTreeSet::new(); + for (entity_type, ids) in by_type { + let entity_ids = ids.into_iter().collect::>(); + let Some(rows) = query_plane + .load_entity_catalog_rows(tenant.as_str(), &entity_type, &entity_ids) + .await + .map_err(|e| { + DispatchError::Internal(format!( + "query projection preflight failed for composite {entity_type} creates: {e}" + )) + })? + else { + continue; + }; + + let present = rows + .into_iter() + .map(|row| row.entity_id) + .collect::>(); + for entity_id in entity_ids { + if !present.contains(&entity_id) { + absent.insert((entity_type.clone(), entity_id)); + } + } + } + + Ok(absent) + } + + async fn preflight_composite_sub_write_transition( + &self, + tenant: &TenantId, + parent_entity_type: &str, + parent_action: &str, + write: &PreparedCompositeSubWrite, + known_absent_create: bool, + ) -> Result { + let table = self.transition_table_for_dispatch(tenant, &write.entity_type)?; + let known_absent_create = known_absent_create + && write.action == "Create" + && !self.entity_exists(tenant, &write.entity_type, &write.entity_id); + let target_exists = if known_absent_create { + false + } else { + self.ensure_entity_loaded(tenant, &write.entity_type, &write.entity_id) + .await + }; + let target_state = if known_absent_create { + synthetic_initial_state(&write.entity_type, &write.entity_id, &table) + } else if target_exists { + self.get_tenant_entity_state(tenant, &write.entity_type, &write.entity_id) + .await + .map_err(DispatchError::Internal)? + .state + } else { + synthetic_initial_state(&write.entity_type, &write.entity_id, &table) + }; + + let preflight_target = PreflightCompositeTarget { + target_existed: target_exists, + state: target_state.clone(), + }; + + if target_state.has_processed_idempotency_key(&write.idempotency_key) { + return Ok(preflight_target); + } + + validate_composite_ref_preflight_compare_and_set( + parent_entity_type, + parent_action, + write, + &preflight_target, + )?; + + if !target_state.can_accept_event() { + return Err(DispatchError::Internal(format!( + "composite {parent_entity_type}.{parent_action} sub-write {} would exceed the event budget for {}:{}", + write.idx, write.entity_type, write.entity_id + ))); + } + + let cross_entity_booleans = + if table_has_cross_entity_guards_for_action(&table, &write.action) { + self.resolve_cross_entity_guards( + tenant, + &write.entity_type, + &write.entity_id, + &write.action, + ) + .await + } else { + BTreeMap::new() + }; + let eval_ctx = build_eval_context_with_xref(&target_state, &cross_entity_booleans); + match table.evaluate_ctx(&target_state.status, &eval_ctx, &write.action) { + Some(result) if result.success => Ok(preflight_target), + Some(_) => Err(DispatchError::Conflict(format!( + "composite {parent_entity_type}.{parent_action} sub-write {} would fail: action '{}' is not valid from state '{}'", + write.idx, write.action, target_state.status + ))), + None => Err(DispatchError::Internal(format!( + "composite {parent_entity_type}.{parent_action} sub-write {} would fail: unknown action '{}'", + write.idx, write.action + ))), + } + } + + async fn composite_sub_write_auth_resource_attrs( + &self, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + action: &str, + params: &Value, + ) -> Result, DispatchError> { + if action == "Create" { + return self.composite_create_resource_attrs(tenant, entity_type, entity_id, params); + } + + if !self + .ensure_entity_loaded(tenant, entity_type, entity_id) + .await + { + return Err(DispatchError::Internal(format!( + "composite sub-write target {entity_type}:{entity_id} does not exist" + ))); + } + + self.load_authz_resource_snapshot(tenant, entity_type, entity_id) + .await + .map(|snapshot| snapshot.resource_attrs) + .map_err(DispatchError::Internal) + } + + fn composite_create_auth_defaults( + &self, + tenant: &TenantId, + entity_type: &str, + ) -> Result { + let table = self.transition_table_for_dispatch(tenant, entity_type)?; + let has_spec = self + .has_registered_spec(tenant, entity_type) + .map_err(DispatchError::Internal)?; + Ok(CompositeCreateAuthDefaults { + initial_state: table.initial_state.clone(), + has_spec, + }) + } + + fn composite_create_resource_attrs( + &self, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + params: &Value, + ) -> Result, DispatchError> { + let table = self.transition_table_for_dispatch(tenant, entity_type)?; + let mut resource_attrs = BTreeMap::new(); + resource_attrs.insert("id".to_string(), Value::String(entity_id.to_string())); + resource_attrs.insert( + "status".to_string(), + Value::String(table.initial_state.clone()), + ); + if let Value::Object(fields) = params { + for (key, value) in fields { + resource_attrs.insert(key.clone(), value.clone()); + } + } + let has_spec = self + .has_registered_spec(tenant, entity_type) + .map_err(DispatchError::Internal)?; + resource_attrs.insert("has_spec".to_string(), Value::Bool(has_spec)); + Ok(resource_attrs) + } + + pub(super) fn transition_table_for_dispatch( + &self, + tenant: &TenantId, + entity_type: &str, + ) -> Result, DispatchError> { + if let Some(table) = self + .registry + .read() + .map_err(|e| DispatchError::Internal(format!("registry lock poisoned: {e}")))? + .get_table(tenant, entity_type) + { + return Ok(table); + } + + self.transition_tables + .get(entity_type) + .cloned() + .ok_or_else(|| DispatchError::Ungoverned(entity_type.to_string())) + } + + #[allow(dead_code)] + async fn ensure_composite_entry_transition_allowed( + &self, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + action: &str, + ) -> Result<(), DispatchError> { + let table = self.transition_table_for_dispatch(tenant, entity_type)?; + let current = self + .get_tenant_entity_state(tenant, entity_type, entity_id) + .await + .map_err(DispatchError::Internal)?; + let cross_entity_booleans = self + .resolve_cross_entity_guards(tenant, entity_type, entity_id, action) + .await; + let eval_ctx = build_eval_context_with_xref(¤t.state, &cross_entity_booleans); + + match table.evaluate_ctx(¤t.state.status, &eval_ctx, action) { + Some(result) if result.success => Ok(()), + Some(_) => Err(DispatchError::Internal(format!( + "Composite action '{action}' not valid from state '{}'", + current.state.status + ))), + None => Err(DispatchError::Internal(format!( + "Unknown composite action: {action}" + ))), + } + } +} diff --git a/crates/temper-server/src/state/dispatch/composite/projection.rs b/crates/temper-server/src/state/dispatch/composite/projection.rs index 2192696d4..ef16ffb9b 100644 --- a/crates/temper-server/src/state/dispatch/composite/projection.rs +++ b/crates/temper-server/src/state/dispatch/composite/projection.rs @@ -21,7 +21,10 @@ impl ServerState { .and_then(|slot| slot.clone()); let mut projection_upserts = Vec::new(); - for stream in streams.values().filter(|stream| !stream.events.is_empty()) { + // A durable exact retry carries no new events, but it is also the + // recovery path for an ambiguous post-commit failure. Re-project every + // captured participant so the retry completes cache/query convergence. + for stream in streams.values() { self.cache_entity_status( format!("{tenant}:{}:{}", stream.entity_type, stream.entity_id), stream.state.status.clone(), diff --git a/crates/temper-server/src/state/dispatch/composite_test.rs b/crates/temper-server/src/state/dispatch/composite_test.rs index 24e6c9842..918b269bd 100644 --- a/crates/temper-server/src/state/dispatch/composite_test.rs +++ b/crates/temper-server/src/state/dispatch/composite_test.rs @@ -2,6 +2,8 @@ use std::collections::BTreeMap; use serde_json::json; use temper_runtime::ActorSystem; +#[cfg(feature = "sim")] +use temper_runtime::persistence::{EventStore, PersistenceError}; use temper_spec::csdl::parse_csdl; #[cfg(feature = "sim")] use temper_store_sim::SimEventStore; @@ -215,6 +217,18 @@ params = ["Reason"] target_entity = "Child" action = "Create" generated_from = "child" + +[[action]] +name = "CreateChildWithEffect" +kind = "Composite" +from = ["Active"] +to = "Active" +params = ["Reason"] + +[[action.sub_writes]] +target_entity = "Child" +action = "CreateWithEffect" +generated_from = "child" "#; const CHILD_IOA: &str = r#" @@ -223,12 +237,22 @@ name = "Child" states = ["Draft", "Active", "Deleted"] initial = "Draft" +[[state]] +name = "revision" +type = "counter" +initial = "0" + +[[key]] +name = "child_name" +properties = ["Name"] + [[action]] name = "Create" kind = "input" from = ["Draft"] to = "Active" params = ["Name"] +effect = [{ type = "increment", var = "revision" }] [[action]] name = "Delete" @@ -236,6 +260,14 @@ kind = "input" from = ["Active"] to = "Deleted" params = [] + +[[action]] +name = "CreateWithEffect" +kind = "input" +from = ["Draft"] +to = "Active" +params = ["Name"] +effect = [{ type = "schedule", action = "Delete", delay_seconds = 2700 }] "#; const APP_IOA: &str = r#" @@ -346,6 +378,200 @@ fn composite_test_state_with_store(store: SimEventStore) -> ServerState { .expect("test state should build") } +#[cfg(feature = "sim")] +#[derive(Clone)] +struct PauseFirstAtomicStableLoadStore { + inner: SimEventStore, + target_boundary_calls: std::sync::Arc, + reached: std::sync::Arc, + resume: std::sync::Arc, +} + +#[cfg(feature = "sim")] +impl PauseFirstAtomicStableLoadStore { + fn new(inner: SimEventStore) -> Self { + Self { + inner, + target_boundary_calls: std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)), + reached: std::sync::Arc::new(tokio::sync::Notify::new()), + resume: std::sync::Arc::new(tokio::sync::Notify::new()), + } + } + + async fn wait_until_first_atomic_load_is_paused(&self) { + self.reached.notified().await; + } + + fn resume_first_atomic_load(&self) { + self.resume.notify_one(); + } +} + +#[cfg(feature = "sim")] +impl EventStore for PauseFirstAtomicStableLoadStore { + fn append( + &self, + persistence_id: &str, + expected_sequence: u64, + events: &[temper_runtime::persistence::PersistenceEnvelope], + ) -> impl std::future::Future> + Send { + self.inner.append(persistence_id, expected_sequence, events) + } + + fn append_batch( + &self, + appends: &[temper_runtime::persistence::PersistenceAppend], + ) -> impl std::future::Future< + Output = Result< + Vec, + PersistenceError, + >, + > + Send { + self.inner.append_batch(appends) + } + + fn batch_idempotency_committed( + &self, + claim: &temper_runtime::persistence::PersistenceBatchIdempotency, + ) -> impl std::future::Future> + Send { + self.inner.batch_idempotency_committed(claim) + } + + fn read_events( + &self, + persistence_id: &str, + from_sequence: u64, + ) -> impl std::future::Future< + Output = Result, PersistenceError>, + > + Send { + self.inner.read_events(persistence_id, from_sequence) + } + + fn read_events_page( + &self, + persistence_id: &str, + from_sequence: u64, + through_sequence: u64, + limit: usize, + ) -> impl std::future::Future< + Output = Result, PersistenceError>, + > + Send { + self.inner + .read_events_page(persistence_id, from_sequence, through_sequence, limit) + } + + async fn journal_boundary( + &self, + persistence_id: &str, + ) -> Result { + if persistence_id == "default:Child:concurrent-exact-child" { + let call = self + .target_boundary_calls + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + if call == 2 { + self.reached.notify_one(); + self.resume.notified().await; + } + } + self.inner.journal_boundary(persistence_id).await + } + + fn save_snapshot( + &self, + persistence_id: &str, + sequence_nr: u64, + snapshot: &[u8], + ) -> impl std::future::Future> + Send { + self.inner + .save_snapshot(persistence_id, sequence_nr, snapshot) + } + + fn save_snapshot_if_source( + &self, + persistence_id: &str, + sequence_nr: u64, + snapshot: &[u8], + source: &temper_runtime::persistence::SnapshotSourceFence, + key_contract: Option<&str>, + ) -> impl std::future::Future> + Send { + self.inner.save_snapshot_if_source( + persistence_id, + sequence_nr, + snapshot, + source, + key_contract, + ) + } + + fn load_snapshot( + &self, + persistence_id: &str, + ) -> impl std::future::Future)>, PersistenceError>> + Send + { + self.inner.load_snapshot(persistence_id) + } + + fn list_entity_ids( + &self, + tenant: &str, + ) -> impl std::future::Future, PersistenceError>> + Send + { + self.inner.list_entity_ids(tenant) + } + + fn list_entity_ids_by_type( + &self, + tenant: &str, + entity_type: &str, + ) -> impl std::future::Future, PersistenceError>> + Send { + self.inner.list_entity_ids_by_type(tenant, entity_type) + } +} + +#[cfg(feature = "sim")] +fn composite_test_state_with_paused_atomic_load_store( + store: PauseFirstAtomicStableLoadStore, +) -> ServerState { + let mut state = composite_test_state(); + state.set_storage_stack(StorageStack::new( + crate::storage::BackendLabel::Sim, + crate::storage::BoxedEventStore::new(store), + None, + None, + None, + None, + None, + None, + None, + None, + )); + state +} + +#[cfg(feature = "sim")] +fn composite_registry_test_state_with_store(store: SimEventStore) -> ServerState { + let csdl = parse_csdl(COMPOSITE_CSDL).expect("test CSDL should parse"); + let mut registry = crate::registry::SpecRegistry::new(); + registry.register_tenant( + "default", + csdl, + COMPOSITE_CSDL.to_string(), + &[ + ("Parent", PARENT_IOA), + ("Child", CHILD_IOA), + ("App", APP_IOA), + ("Blob", BLOB_IOA), + ("Ref", REF_IOA), + ], + ); + let mut state = ServerState::from_registry( + ActorSystem::new("composite-registry-dispatch-test"), + registry, + ); + state.set_storage_stack(StorageStack::from_sim(store, None)); + state +} + #[tokio::test] async fn composite_action_rejects_caller_supplied_sub_writes() { let state = composite_test_state(); @@ -949,6 +1175,628 @@ async fn composite_atomic_batch_records_parent_composite_event_once() { ); } +#[cfg(feature = "sim")] +#[tokio::test] +async fn composite_exact_retry_is_content_bound_and_repairs_runtime_convergence() { + let store = SimEventStore::no_faults(401); + let state = composite_test_state_with_store(store.clone()); + let tenant = TenantId::default(); + let mut agent = AgentContext::for_service("composite-retry-test"); + agent.idempotency_key = Some("stable-parent-operation".to_string()); + let child_id = "child-content-bound-retry"; + let callback = json!({ + "sub_writes": [{ + "entity_type": "Child", + "entity_id": child_id, + "action": "Create", + "params": { "Name": "original value" } + }] + }); + state + .apply_composite_integration_result( + &tenant, + "Parent", + "parent-content-bound-retry", + "CreateChild", + &callback, + &agent, + ) + .await + .expect("first composite must commit"); + let child_pid = format!("default:Child:{child_id}"); + let first_journal_len = store.dump_journal(&child_pid).len(); + + state.cache_entity_status(child_pid.clone(), "StaleProjection".to_string()); + state + .apply_composite_integration_result( + &tenant, + "Parent", + "parent-content-bound-retry", + "CreateChild", + &callback, + &agent, + ) + .await + .expect("exact retry must finish post-commit convergence"); + assert_eq!(store.dump_journal(&child_pid).len(), first_journal_len); + assert_eq!( + state + .entity_state_cache + .lock() + .expect("state cache lock") + .get(&child_pid) + .map(|(status, _)| status.as_str()), + Some("Active"), + "an exact durable retry must refresh derived runtime state" + ); + + let error = state + .apply_composite_integration_result( + &tenant, + "Parent", + "parent-content-bound-retry", + "CreateChild", + &json!({ + "sub_writes": [{ + "entity_type": "Child", + "entity_id": child_id, + "action": "Create", + "params": { "Name": "different value" } + }] + }), + &agent, + ) + .await + .expect_err("one parent idempotency key cannot authorize different sub-write values"); + assert!(error.to_string().contains("different intent")); + assert_eq!(store.dump_journal(&child_pid).len(), first_journal_len); +} + +#[cfg(feature = "sim")] +#[tokio::test] +async fn composite_exact_retry_uses_durable_claim_after_actor_history_ages_out() { + let store = SimEventStore::no_faults(402); + let state = composite_test_state_with_store(store.clone()); + let tenant = TenantId::default(); + let mut agent = AgentContext::for_service("composite-aged-retry-test"); + agent.idempotency_key = Some("aged-parent-operation".to_string()); + let child_id = "child-aged-content-bound-retry"; + let callback = json!({ + "sub_writes": [{ + "entity_type": "Child", + "entity_id": child_id, + "action": "Create", + "params": { "Name": "durable value" } + }] + }); + + state + .apply_composite_integration_result( + &tenant, + "Parent", + "parent-aged-content-bound-retry", + "CreateChild", + &callback, + &agent, + ) + .await + .expect("first composite must commit"); + let child_pid = format!("default:Child:{child_id}"); + let expected_sequence = store.dump_journal(&child_pid).len() as u64; + let aged_events = (0..=crate::entity_actor::types::MAX_DURABLE_IDEMPOTENCY_KEYS_PER_ENTITY) + .map(|index| { + composite_envelope( + &child_pid, + &crate::entity_actor::EntityEvent { + action: "AgedHistory".to_string(), + from_status: "Active".to_string(), + to_status: "Active".to_string(), + timestamp: temper_runtime::scheduler::sim_now(), + params: json!({"index": index}), + idempotency_key: Some(format!("aged-history-{index}")), + }, + ) + .expect("encode aged event") + }) + .collect::>(); + store + .append(&child_pid, expected_sequence, &aged_events) + .await + .expect("advance child beyond bounded actor idempotency history"); + state.stop_and_remove_entity(&tenant, "Child", child_id); + state.cache_entity_status(child_pid.clone(), "StaleProjection".to_string()); + let journal_len = store.dump_journal(&child_pid).len(); + + state + .apply_composite_integration_result( + &tenant, + "Parent", + "parent-aged-content-bound-retry", + "CreateChild", + &callback, + &agent, + ) + .await + .expect("durable claim must bypass current-state guards after actor history eviction"); + + assert_eq!(store.dump_journal(&child_pid).len(), journal_len); + assert_eq!( + state + .entity_state_cache + .lock() + .expect("state cache lock") + .get(&child_pid) + .map(|(status, _)| status.as_str()), + Some("Active"), + "exact replay must still repair derived runtime convergence" + ); +} + +#[cfg(feature = "sim")] +#[tokio::test] +async fn malformed_effectful_composite_reservation_fails_closed() { + let store = SimEventStore::no_faults(4_012); + let state = composite_test_state_with_store(store.clone()); + let tenant = TenantId::default(); + let agent = AgentContext::for_service("malformed-composite-reservation-test"); + let parent_idempotency = "malformed-effectful-reservation"; + let parent = AtomicCompositeParent { + tenant: &tenant, + entity_type: "Parent", + entity_id: "parent-malformed-reservation", + action: "CreateChildWithEffect", + idempotency: parent_idempotency, + record_event: true, + agent_ctx: &agent, + }; + let mut intended = build_composite_event( + &tenant, + parent.entity_type, + parent.entity_id, + parent.action, + parent.idempotency, + &[], + ); + intended.intent_hash = "exact-intent".to_string(); + let persistence_id = + ServerState::effectful_composite_reservation_persistence_id(parent, &intended); + store + .append( + &persistence_id, + 0, + &[PersistenceEnvelope { + sequence_nr: 0, + event_type: COMPOSITE_EVENT_TYPE.to_string(), + payload: json!({"schema": "malformed-composite-reservation"}), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: sim_now(), + actor_id: persistence_id.clone(), + }, + }], + ) + .await + .expect("seed malformed reserved composite intent"); + let (event_store, _) = state.event_journal().expect("sim event journal"); + + let error = state + .effectful_composite_reservation_exists(&event_store, parent, &intended) + .await + .expect_err("reserved composite corruption must block effect replay"); + assert!( + error + .to_string() + .contains("malformed composite reservation") + ); +} + +#[cfg(feature = "sim")] +#[tokio::test] +async fn composite_effectful_subwrite_is_durably_content_bound() { + let store = SimEventStore::no_faults(403); + let state = composite_test_state_with_store(store.clone()); + let tenant = TenantId::default(); + let mut agent = AgentContext::for_service("composite-effect-retry-test"); + agent.idempotency_key = Some("effectful-parent-operation".to_string()); + let child_id = "child-effect-content-bound"; + let child_pid = format!("default:Child:{child_id}"); + let bootstrap = crate::entity_actor::EntityEvent { + action: "Created".to_string(), + from_status: String::new(), + to_status: "Draft".to_string(), + timestamp: sim_now(), + params: json!({}), + idempotency_key: None, + }; + store + .append( + &child_pid, + 0, + &[composite_envelope(&child_pid, &bootstrap) + .expect("encode effectful child bootstrap")], + ) + .await + .expect("seed an existing Draft child for the non-Create action"); + let callback = json!({ + "sub_writes": [{ + "entity_type": "Child", + "entity_id": child_id, + "action": "CreateWithEffect", + "params": { "Name": "original effect value" } + }] + }); + let metadata = state + .composite_metadata_for(&tenant, "Parent", "CreateChildWithEffect") + .expect("load composite metadata") + .expect("effectful composite metadata"); + let parent_idempotency = composite_parent_idempotency(&agent, &callback); + let parent = AtomicCompositeParent { + tenant: &tenant, + entity_type: "Parent", + entity_id: "parent-effect-content-bound", + action: "CreateChildWithEffect", + idempotency: &parent_idempotency, + record_event: metadata.record_parent_event, + agent_ctx: &agent, + }; + let sub_writes = parse_sub_writes(&callback).expect("parse effectful sub-write"); + let batch_claim = composite_batch_claim( + parent, + &prepare_composite_intent_sub_writes(parent, &sub_writes, &metadata), + ) + .expect("build effectful batch claim"); + + state + .apply_composite_integration_result( + &tenant, + "Parent", + "parent-effect-content-bound", + "CreateChildWithEffect", + &callback, + &agent, + ) + .await + .expect("effectful composite must commit atomically"); + assert!( + !store + .batch_idempotency_committed(&batch_claim) + .await + .expect("read effectful batch claim"), + "non-durable post-commit effects must use actor idempotency instead of an atomic batch claim" + ); + let journal_len = store.dump_journal(&child_pid).len(); + + let error = state + .apply_composite_integration_result( + &tenant, + "Parent", + "parent-effect-content-bound", + "CreateChildWithEffect", + &json!({ + "sub_writes": [{ + "entity_type": "Child", + "entity_id": child_id, + "action": "CreateWithEffect", + "params": { "Name": "different effect value" } + }] + }), + &agent, + ) + .await + .expect_err("effectful retry with different content must conflict"); + assert!(error.to_string().contains("different intent")); + assert_eq!(store.dump_journal(&child_pid).len(), journal_len); +} + +#[cfg(feature = "sim")] +#[tokio::test] +async fn delayed_composite_batch_cannot_borrow_reactivated_key_epoch() { + let store = SimEventStore::no_faults(299); + let state = composite_registry_test_state_with_store(store.clone()); + let tenant = TenantId::default(); + let child_live_table = state + .registry + .read() + .expect("registry lock") + .get_table_live(&tenant, "Child") + .expect("Child table"); + let original_table = child_live_table.read().expect("table lock").clone(); + let signature_a = crate::key_index::declared_key_set_signature(&original_table.keys); + let old_epoch = store + .activate_key_index_contract(tenant.as_str(), "Child", &signature_a, false) + .await + .expect("activate original Child contract"); + store + .mark_key_index_backfilled(tenant.as_str(), "Child", &signature_a) + .await + .expect("publish original Child readiness"); + child_live_table + .write() + .expect("table lock") + .key_contract_activation_epoch = old_epoch; + + let callback_params = json!({ + "sub_writes": [{ + "entity_type": "Child", + "entity_id": "child-stale-composite", + "action": "Create", + "params": { "Name": "must-not-resurrect" } + }] + }); + let agent = AgentContext::for_service("composite-epoch-test"); + let pause = store.inject_precommit_batch_pause(); + let composite_future = state.apply_composite_integration_result( + &tenant, + "Parent", + "parent-stale-composite", + "CreateChild", + &callback_params, + &agent, + ); + tokio::pin!(composite_future); + tokio::select! { + result = &mut composite_future => panic!("composite crossed pre-commit barrier: {result:?}"), + () = pause.wait_until_reached() => {} + } + + let signature_none = crate::key_index::declared_key_set_signature(&[]); + let empty_epoch = store + .activate_key_index_contract(tenant.as_str(), "Child", &signature_none, true) + .await + .expect("activate empty Child contract"); + let mut empty_table = original_table.clone(); + empty_table.keys.clear(); + empty_table.key_contract_activation_epoch = empty_epoch; + *child_live_table.write().expect("table lock") = empty_table; + + let current_epoch = store + .activate_key_index_contract(tenant.as_str(), "Child", &signature_a, false) + .await + .expect("reactivate Child contract"); + let mut current_table = original_table; + current_table.key_contract_activation_epoch = current_epoch; + *child_live_table.write().expect("table lock") = current_table; + + pause.resume(); + let error = composite_future + .await + .expect_err("staged old-epoch composite must reject atomically"); + assert!(error.to_string().contains("activation is stale")); + assert!( + store + .dump_journal("default:Parent:parent-stale-composite") + .is_empty() + ); + assert!( + store + .dump_journal("default:Child:child-stale-composite") + .is_empty() + ); +} + +#[cfg(feature = "sim")] +#[tokio::test] +async fn composite_parent_audit_respects_the_raw_replay_tail_budget() { + let store = SimEventStore::no_faults(296); + let parent_pid = "default:Parent:parent-at-budget"; + let events = (0..crate::entity_actor::types::MAX_EVENTS_SINCE_SNAPSHOT) + .map(|_| { + let event = crate::entity_actor::EntityEvent { + action: "CreateChild".to_string(), + from_status: "Active".to_string(), + to_status: "Active".to_string(), + timestamp: sim_now(), + params: json!({}), + idempotency_key: None, + }; + PersistenceEnvelope { + sequence_nr: 0, + event_type: event.action.clone(), + payload: serde_json::to_value(event).expect("serialize parent event"), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: sim_now(), + actor_id: parent_pid.to_string(), + }, + } + }) + .collect::>(); + store + .append(parent_pid, 0, &events) + .await + .expect("seed parent at raw replay-tail cap"); + + let state = composite_test_state_with_store(store.clone()); + let error = state + .apply_composite_integration_result( + &TenantId::default(), + "Parent", + "parent-at-budget", + "CreateChild", + &json!({ + "sub_writes": [{ + "entity_type": "Child", + "entity_id": "child-must-not-append", + "action": "Create", + "params": {"Name": "blocked by parent budget"} + }] + }), + &AgentContext::for_service("composite-test"), + ) + .await + .expect_err("parent audit at the cap must reject the whole composite") + .to_string(); + + assert!(error.contains("parent audit would exceed the event budget")); + assert_eq!( + store.dump_journal(parent_pid).len(), + crate::entity_actor::types::MAX_EVENTS_SINCE_SNAPSHOT + ); + assert!( + store + .dump_journal("default:Child:child-must-not-append") + .is_empty(), + "the atomic child write must not commit after the parent budget rejects" + ); +} + +#[cfg(feature = "sim")] +#[tokio::test] +async fn composite_first_write_materializes_snapshot_only_counter_for_restart() { + let store = SimEventStore::no_faults(288); + let tenant = TenantId::default(); + let child_id = "snapshot-only-composite-child"; + let child_pid = format!("default:Child:{child_id}"); + let snapshot = serde_json::to_vec(&json!({ + "entity_type": "Child", + "entity_id": child_id, + "status": "Draft", + "item_count": 0, + "counters": {"revision": 10}, + "booleans": {}, + "lists": {}, + "fields": { + "Id": "legacy-wrong-child-id", + "Name": "snapshot baseline", + "Status": "LegacyWrongStatus" + }, + "events": [], + "total_event_count": 10, + "events_since_snapshot": 0, + "last_snapshot_sequence_nr": 5, + "sequence_nr": 5, + "processed_idempotency_keys": {} + })) + .expect("serialize snapshot-only composite target"); + store + .save_snapshot(&child_pid, 5, &snapshot) + .await + .expect("seed snapshot-only composite target"); + + let state = composite_test_state_with_store(store.clone()); + state + .apply_composite_integration_result( + &tenant, + "Parent", + "snapshot-only-composite-parent", + "CreateChild", + &json!({ + "sub_writes": [{ + "entity_type": "Child", + "entity_id": child_id, + "action": "Create", + "params": {"Name": "after composite"} + }] + }), + &AgentContext::for_service("composite-test"), + ) + .await + .expect("apply composite write to snapshot-only target"); + + let journal = store.dump_journal(&child_pid); + assert_eq!( + journal + .iter() + .map(|event| event.event_type.as_str()) + .collect::>(), + vec!["Temper.Internal.StateMaterialization.v1", "Create"] + ); + assert_eq!(journal[0].payload["state"]["fields"]["Id"], child_id); + assert_eq!(journal[0].payload["state"]["fields"]["Status"], "Draft"); + let restarted = composite_test_state_with_store(store); + let child = restarted + .get_tenant_entity_state(&tenant, "Child", child_id) + .await + .expect("restart snapshot-only composite target"); + assert_eq!(child.state.counters.get("revision"), Some(&11)); + assert_eq!( + child.state.fields.get("Name"), + Some(&json!("after composite")) + ); + assert_eq!(child.state.sequence_nr, 2); + assert_eq!(child.state.fields["Id"], child_id); + assert_eq!(child.state.fields["Status"], "Active"); +} + +#[cfg(feature = "sim")] +#[tokio::test] +async fn snapshot_only_parent_composite_restart_does_not_fabricate_created_event() { + let store = SimEventStore::no_faults(290); + let tenant = TenantId::default(); + let parent_id = "snapshot-only-composite-parent"; + let parent_pid = format!("default:Parent:{parent_id}"); + let snapshot = serde_json::to_vec(&json!({ + "entity_type": "Parent", + "entity_id": parent_id, + "status": "Active", + "item_count": 0, + "counters": {}, + "booleans": {}, + "lists": {}, + "fields": {"Id": parent_id, "Status": "Active"}, + "events": [], + "total_event_count": 0, + "events_since_snapshot": 0, + "last_snapshot_sequence_nr": 5, + "sequence_nr": 5, + "processed_idempotency_keys": {} + })) + .expect("serialize snapshot-only parent"); + store + .save_snapshot(&parent_pid, 5, &snapshot) + .await + .expect("seed snapshot-only parent"); + + let state = composite_test_state_with_store(store.clone()); + state + .apply_composite_integration_result( + &tenant, + "Parent", + parent_id, + "CreateChild", + &json!({ + "sub_writes": [{ + "entity_type": "Child", + "entity_id": "snapshot-parent-child", + "action": "Create", + "params": {"Name": "child"} + }] + }), + &AgentContext::for_service("composite-test"), + ) + .await + .expect("apply composite against snapshot-only parent"); + + assert_eq!( + store + .dump_journal(&parent_pid) + .iter() + .map(|event| event.event_type.as_str()) + .collect::>(), + vec![ + "Temper.Internal.StateMaterialization.v1", + COMPOSITE_EVENT_TYPE + ], + "reloading a materialized parent with only composite audit history must not bootstrap Created" + ); + let restarted = composite_test_state_with_store(store.clone()); + let parent = restarted + .get_tenant_entity_state(&tenant, "Parent", parent_id) + .await + .expect("restart snapshot-only composite parent"); + assert_eq!(parent.state.sequence_nr, 2); + assert_eq!(parent.state.total_event_count, 0); + assert_eq!( + store.dump_journal(&parent_pid).len(), + 2, + "a second restart must not fabricate a domain Created event" + ); +} + #[cfg(feature = "sim")] #[tokio::test] async fn composite_atomic_batch_can_skip_parent_composite_event_by_spec() { @@ -1374,6 +2222,22 @@ async fn composite_atomic_batch_allows_existing_sub_write_to_delete_target() { .expect("child create should run"); assert!(created.success); assert!(state.entity_exists(&tenant, "Child", child_id)); + let child_name_hash = crate::key_index::canonical_key_hash( + "child_name", + &["Name".to_string()], + json!({ "Name": "temporary child" }) + .as_object() + .expect("key fields object"), + ) + .expect("complete child name key"); + assert_eq!( + store + .lookup_by_key(tenant.as_str(), "Child", "child_name", &child_name_hash,) + .await + .expect("lookup created child key"), + Some(child_id.to_string()), + "precondition: normal create owns the declared key" + ); let applied = state .apply_composite_integration_result( @@ -1400,6 +2264,14 @@ async fn composite_atomic_batch_allows_existing_sub_write_to_delete_target() { "deleted composite sub-write target should not be reloaded as a live entity" ); assert!(!state.entity_exists(&tenant, "Child", child_id)); + assert_eq!( + store + .lookup_by_key(tenant.as_str(), "Child", "child_name", &child_name_hash,) + .await + .expect("lookup key after composite delete"), + None, + "composite delete must release declared-key ownership atomically" + ); let child_journal = store.dump_journal(&format!("default:Child:{child_id}")); assert_eq!( @@ -1593,6 +2465,94 @@ async fn composite_atomic_batch_handles_concurrent_multi_entity_results() { } } +#[cfg(feature = "sim")] +#[tokio::test] +async fn concurrent_exact_composite_retries_converge_on_the_durable_claim() { + let inner = SimEventStore::no_faults(406); + let store = PauseFirstAtomicStableLoadStore::new(inner.clone()); + let state = composite_test_state_with_paused_atomic_load_store(store.clone()); + let tenant = TenantId::default(); + let mut agent = AgentContext::for_service("concurrent-exact-composite-test"); + agent.idempotency_key = Some("concurrent-exact-parent-key".to_string()); + let callback = json!({ + "sub_writes": [{ + "entity_type": "Child", + "entity_id": "concurrent-exact-child", + "action": "Create", + "params": {"Name": "committed exactly once"} + }] + }); + + let paused_state = state.clone(); + let paused_tenant = tenant.clone(); + let paused_agent = agent.clone(); + let paused_callback = callback.clone(); + let paused = tokio::spawn(async move { + paused_state + .apply_composite_integration_result( + &paused_tenant, + "Parent", + "concurrent-exact-parent", + "CreateChild", + &paused_callback, + &paused_agent, + ) + .await + }); + store.wait_until_first_atomic_load_is_paused().await; + + let winner_state = state.clone(); + let winner_tenant = tenant.clone(); + let winner_agent = agent.clone(); + let winner_callback = callback.clone(); + let winner = tokio::spawn(async move { + winner_state + .apply_composite_integration_result( + &winner_tenant, + "Parent", + "concurrent-exact-parent", + "CreateChild", + &winner_callback, + &winner_agent, + ) + .await + }); + // Without per-claim serialization the peer reaches durable commit while + // this callback is paused after preflight. With serialization it remains + // queued; the bounded yields merely let either deterministic schedule make + // progress before the captured source resumes. + for _ in 0..256 { + if !inner + .dump_journal("default:Child:concurrent-exact-child") + .is_empty() + { + break; + } + tokio::task::yield_now().await; + } + store.resume_first_atomic_load(); + let winner = winner + .await + .expect("concurrent exact callback should join") + .expect("concurrent exact callback should commit or replay"); + assert!(winner); + let replay = paused + .await + .expect("paused exact callback should join") + .expect("paused exact callback must replay the committed durable claim"); + assert!(replay); + + assert_eq!( + inner + .dump_journal("default:Child:concurrent-exact-child") + .iter() + .filter(|event| event.event_type == "Create") + .count(), + 1, + "both successful callbacks must converge on one durable child transition" + ); +} + #[tokio::test] async fn commons_composite_rejects_duplicate_owner_app_name_before_dispatch() { let state = composite_test_state(); diff --git a/crates/temper-server/src/state/dispatch/cross_entity.rs b/crates/temper-server/src/state/dispatch/cross_entity.rs index eb127b68f..0bb84e087 100644 --- a/crates/temper-server/src/state/dispatch/cross_entity.rs +++ b/crates/temper-server/src/state/dispatch/cross_entity.rs @@ -241,7 +241,13 @@ impl crate::state::ServerState { let child_id = req.entity_id.clone(); let initial_action = req.initial_action.clone(); let parent_params = action_params.clone(); - let agent = agent_ctx.clone(); + let agent = agent_ctx + .fork_tenant_generation_lease(tenant) + .unwrap_or_else(|| { + let mut detached = agent_ctx.clone(); + detached.detach_tenant_generation_lease(); + detached + }); let copied_fields = req.copied_field_values.clone(); let workflow_root_entity_type = agent .workflow_root_entity_type @@ -269,6 +275,15 @@ impl crate::state::ServerState { tokio::spawn( async move { // determinism-ok: spawn dispatch is a background side-effect + let Some(agent) = state.activate_immediate_tenant_work(&t, agent).await else { + tracing::error!( + tenant = %t, + child_type = %child_type, + child_id = %child_id, + "spawn request could not enter a complete runtime generation" + ); + return; + }; let mut parent_fields = serde_json::Map::new(); parent_fields.insert( "parent_type".to_string(), diff --git a/crates/temper-server/src/state/dispatch/effects.rs b/crates/temper-server/src/state/dispatch/effects.rs index 49a612641..feb94c887 100644 --- a/crates/temper-server/src/state/dispatch/effects.rs +++ b/crates/temper-server/src/state/dispatch/effects.rs @@ -520,7 +520,8 @@ impl crate::state::ServerState { let et = entity_type.to_string(); let eid = entity_id.to_string(); let action = sched.action.clone(); - let ctx = agent_ctx.clone(); + let mut ctx = agent_ctx.clone(); + ctx.detach_tenant_generation_lease(); let delay = std::time::Duration::from_secs(sched.delay_seconds); let workflow_root_entity_type = ctx .workflow_root_entity_type @@ -547,6 +548,16 @@ impl crate::state::ServerState { // determinism-ok: timer delivery is a background side-effect async move { tokio::time::sleep(delay).await; // determinism-ok: scheduled delay + let Some(ctx) = state.activate_immediate_tenant_work(&t, ctx).await else { + tracing::error!( + tenant = %t, + entity_type = %et, + entity_id = %eid, + action = %action, + "scheduled action could not enter a complete runtime generation" + ); + return; + }; let _ = state .dispatch_tenant_action( &t, @@ -727,13 +738,16 @@ impl crate::state::ServerState { && let Some(handler) = &self.custom_effect_handler { for effect_name in &response.custom_effects { - if let Err(e) = handler.handle( - effect_name, - ctx.entity_type, - ctx.entity_id, - &response.state.fields, - self, - ) { + if let Err(e) = handler + .handle( + effect_name, + ctx.entity_type, + ctx.entity_id, + &response.state.fields, + self, + ) + .await + { tracing::error!( effect = %effect_name, entity_type = ctx.entity_type, @@ -741,6 +755,17 @@ impl crate::state::ServerState { error = %e, "custom effect handler failed" ); + return EntityResponse { + success: false, + state: response.state.clone(), + error: Some(format!( + "custom effect '{effect_name}' failed after the action commit: {e}" + )), + custom_effects: response.custom_effects.clone(), + scheduled_actions: response.scheduled_actions.clone(), + spawn_requests: response.spawn_requests.clone(), + spec_governed: response.spec_governed, + }; } } } diff --git a/crates/temper-server/src/state/dispatch/mod.rs b/crates/temper-server/src/state/dispatch/mod.rs index 42026cfde..da85c98da 100644 --- a/crates/temper-server/src/state/dispatch/mod.rs +++ b/crates/temper-server/src/state/dispatch/mod.rs @@ -371,7 +371,13 @@ impl crate::state::ServerState { let action = _action.to_string(); let custom_effects = custom_effects.to_vec(); let entity_state = _entity_state.clone(); - let agent_ctx = agent_ctx.clone(); + let agent_ctx = agent_ctx + .fork_tenant_generation_lease(&tenant) + .unwrap_or_else(|| { + let mut detached = agent_ctx.clone(); + detached.detach_tenant_generation_lease(); + detached + }); let action_params = action_params.clone(); let workflow_root_entity_type = workflow_root_type(&agent_ctx, &entity_type); let workflow_root_entity_id = workflow_root_id(&agent_ctx, &entity_id); @@ -388,6 +394,18 @@ impl crate::state::ServerState { tokio::spawn( async move { // determinism-ok: async integration side-effects run outside simulation core + let Some(agent_ctx) = state + .activate_immediate_tenant_work(&tenant, agent_ctx) + .await + else { + tracing::error!( + tenant = %tenant, + entity_type = %entity_type, + entity_id = %entity_id, + "WASM integrations could not enter a complete runtime generation" + ); + return; + }; let req = WasmDispatchRequest { tenant: &tenant, entity_type: &entity_type, @@ -414,6 +432,7 @@ impl crate::state::ServerState { &entity_id, &action, &e, + &agent_ctx, ); } } @@ -423,69 +442,5 @@ impl crate::state::ServerState { } #[cfg(test)] -mod tests { - use super::*; - use temper_runtime::ActorSystem; - use temper_spec::csdl::parse_csdl; - - fn test_state() -> crate::state::ServerState { - let csdl_xml = include_str!("../../../../../test-fixtures/specs/model.csdl.xml"); - let csdl = parse_csdl(csdl_xml).expect("CSDL should parse"); - crate::state::ServerState::new( - ActorSystem::new("dispatch-wasm-authz-test"), - csdl, - csdl_xml.to_string(), - ) - } - - #[test] - fn wasm_authz_gate_evaluates_cedar_when_policy_set_is_empty() { - let state = test_state(); - state - .authz - .reload_policies("") - .expect("empty policy set should parse"); - - let gate = state.wasm_authz_gate(); - let decision = gate.authorize_http_call( - "api.example.com", - "GET", - "https://api.example.com/v1/ping", - &WasmAuthzContext::test_fixture(), - ); - - assert_eq!( - decision, - WasmAuthzDecision::Deny("no matching permit policy".to_string()) - ); - } - - #[test] - fn wasm_authz_gate_allows_when_cedar_policy_matches() { - let state = test_state(); - state - .authz - .reload_policies( - r#" - permit( - principal is Agent, - action == Action::"http_call", - resource is HttpEndpoint - ) when { - context.module == "stripe_charge" - }; - "#, - ) - .expect("policy should parse"); - - let gate = state.wasm_authz_gate(); - let decision = gate.authorize_http_call( - "api.stripe.com", - "POST", - "https://api.stripe.com/v1/charges", - &WasmAuthzContext::test_fixture(), - ); - - assert_eq!(decision, WasmAuthzDecision::Allow); - } -} +#[path = "mod_test.rs"] +mod tests; diff --git a/crates/temper-server/src/state/dispatch/mod_test.rs b/crates/temper-server/src/state/dispatch/mod_test.rs new file mode 100644 index 000000000..7c5f29ba7 --- /dev/null +++ b/crates/temper-server/src/state/dispatch/mod_test.rs @@ -0,0 +1,64 @@ +use super::*; +use temper_runtime::ActorSystem; +use temper_spec::csdl::parse_csdl; + +fn test_state() -> crate::state::ServerState { + let csdl_xml = include_str!("../../../../../test-fixtures/specs/model.csdl.xml"); + let csdl = parse_csdl(csdl_xml).expect("CSDL should parse"); + crate::state::ServerState::new( + ActorSystem::new("dispatch-wasm-authz-test"), + csdl, + csdl_xml.to_string(), + ) +} + +#[test] +fn wasm_authz_gate_evaluates_cedar_when_policy_set_is_empty() { + let state = test_state(); + state + .authz + .reload_policies("") + .expect("empty policy set should parse"); + + let gate = state.wasm_authz_gate(); + let decision = gate.authorize_http_call( + "api.example.com", + "GET", + "https://api.example.com/v1/ping", + &WasmAuthzContext::test_fixture(), + ); + + assert_eq!( + decision, + WasmAuthzDecision::Deny("no matching permit policy".to_string()) + ); +} + +#[test] +fn wasm_authz_gate_allows_when_cedar_policy_matches() { + let state = test_state(); + state + .authz + .reload_policies( + r#" + permit( + principal is Agent, + action == Action::"http_call", + resource is HttpEndpoint + ) when { + context.module == "stripe_charge" + }; + "#, + ) + .expect("policy should parse"); + + let gate = state.wasm_authz_gate(); + let decision = gate.authorize_http_call( + "api.stripe.com", + "POST", + "https://api.stripe.com/v1/charges", + &WasmAuthzContext::test_fixture(), + ); + + assert_eq!(decision, WasmAuthzDecision::Allow); +} diff --git a/crates/temper-server/src/state/dispatch/state_timeouts.rs b/crates/temper-server/src/state/dispatch/state_timeouts.rs index a6d77ef80..46a84fa13 100644 --- a/crates/temper-server/src/state/dispatch/state_timeouts.rs +++ b/crates/temper-server/src/state/dispatch/state_timeouts.rs @@ -137,6 +137,15 @@ impl StateTimeoutTracker { .unwrap_or(0) } + fn clear_if_current(&self, key: &EntityKey, expected: u64) -> bool { + let mut seqs = self.seqs.lock().expect("state_timeout tracker poisoned"); + if seqs.get(key).copied() != Some(expected) { + return false; + } + seqs.remove(key); + true + } + /// Increment the pending-timer count for `entity_type`. Called at arm. pub fn inc_pending(&self, entity_type: &str) { let mut map = self @@ -221,7 +230,7 @@ impl crate::state::ServerState { .back() .map(|e| e.from_status.clone()) .unwrap_or_default(); - let state_changed = pre_state != post_state; + let state_changed = ctx.action != "__RuntimeGenerationChanged" && pre_state != post_state; let key = EntityKey::new(ctx.tenant, ctx.entity_type, ctx.entity_id); // 1. Invalidate any outstanding timer for the prior state. @@ -318,7 +327,10 @@ impl crate::state::ServerState { let entity_id = ctx.entity_id.to_string(); let target_state = st.state.clone(); let target_action = st.on_timeout.clone(); - let agent_ctx = ctx.agent_ctx.clone(); + let mut agent_ctx = ctx.agent_ctx.clone(); + let armed_generation = agent_ctx + .detach_tenant_generation_lease() + .unwrap_or_else(|| self.tenant_generation_version(ctx.tenant)); let key_for_task = key.clone(); let entity_type_for_dec = ctx.entity_type.to_string(); let workflow_root_entity_type = agent_ctx @@ -365,6 +377,20 @@ impl crate::state::ServerState { ); async move { + let Some(agent_ctx) = state + .activate_immediate_tenant_work(&tenant, agent_ctx) + .await + else { + tracker.dec_pending(&entity_type_for_dec); + tracing::error!( + tenant = %tenant, + entity_type = %entity_type, + entity_id = %entity_id, + target_action = %target_action, + "state timeout could not enter a complete runtime generation" + ); + return; + }; // Sequence-based cancellation check. A newer arm (or a // state change that bumped the seq on exit) renders // this timer a no-op. @@ -373,6 +399,32 @@ impl crate::state::ServerState { return; } + if state.tenant_generation_version(&tenant) != armed_generation { + if !tracker.clear_if_current(&key_for_task, armed_seq) { + tracker.dec_pending(&entity_type_for_dec); + return; + } + tracker.dec_pending(&entity_type_for_dec); + if let Ok(current) = state + .get_tenant_entity_state(&tenant, &entity_type, &entity_id) + .await + { + let refresh_params = serde_json::json!({}); + let refresh_ctx = PostDispatchContext { + tenant: &tenant, + entity_type: &entity_type, + entity_id: &entity_id, + action: "__RuntimeGenerationChanged", + agent_ctx: &agent_ctx, + dispatch_idempotency_key: None, + action_params: &refresh_params, + await_integration: false, + }; + state.arm_state_timeouts_if_needed(&refresh_ctx, ¤t); + } + return; + } + // Secondary check: confirm the entity is still in the // target state. Covers races where the entity left and // re-entered the same state with a new seq greater than @@ -1197,4 +1249,70 @@ queue_timeout_seconds = 30 ); } } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn state_timeout_rearms_under_the_complete_replacement_generation() { + let csdl = parse_csdl(TICKET_CSDL).expect("CSDL parses"); + let mut registry = SpecRegistry::new(); + registry.register_tenant( + "default", + csdl, + TICKET_CSDL.to_string(), + &[("Ticket", TICKET_WITH_TIMEOUT_IOA)], + ); + let state = ServerState::from_registry( + ActorSystem::new("state-timeout-generation-rearm"), + registry, + ); + let tenant = temper_runtime::tenant::TenantId::default(); + let agent_ctx = AgentContext::for_service("timeout-scheduler"); + let response = state + .get_or_create_tenant_entity( + &tenant, + "Ticket", + "rearmed-timeout", + serde_json::json!({}), + ) + .await + .expect("create ticket"); + let ctx = PostDispatchContext { + tenant: &tenant, + entity_type: "Ticket", + entity_id: "rearmed-timeout", + action: "__Created", + agent_ctx: &agent_ctx, + dispatch_idempotency_key: None, + action_params: &serde_json::json!({}), + await_integration: false, + }; + state.arm_state_timeouts_if_needed(&ctx, &response); + + let mut publication = state + .begin_spec_publication(&tenant) + .await + .expect("acquire publication writer"); + let intent = ServerState::spec_publication_intent( + "state-timeout-generation-test", + [("ticket", TICKET_WITH_TIMEOUT_IOA.as_bytes())], + ); + state + .arm_spec_publication(&mut publication, &tenant, &intent) + .expect("arm replacement generation"); + state + .complete_spec_publication(&mut publication, &tenant) + .expect("publish replacement generation"); + drop(publication); + + // A snapshot without retained entry history conservatively receives a + // fresh one-second budget after the generation-aware rearm. + tokio::time::sleep(std::time::Duration::from_secs(3)).await; + let after = state + .get_tenant_entity_state(&tenant, "Ticket", "rearmed-timeout") + .await + .expect("load ticket after generation-aware timeout rearm"); + assert_eq!( + after.state.status, "InProgress", + "the durable timeout obligation must rearm under the replacement generation instead of being dropped" + ); + } } diff --git a/crates/temper-server/src/state/dispatch/wasm.rs b/crates/temper-server/src/state/dispatch/wasm.rs index 708869e1f..abc29c280 100644 --- a/crates/temper-server/src/state/dispatch/wasm.rs +++ b/crates/temper-server/src/state/dispatch/wasm.rs @@ -347,8 +347,12 @@ impl crate::state::ServerState { &self, req: &WasmDispatchRequest<'_>, ) -> Result, String> { + let generation_agent_ctx = self + .dispatch_context_in_generation(req.tenant, req.agent_ctx) + .await + .map_err(|error| error.to_string())?; record_workflow_span_attrs( - req.agent_ctx, + generation_agent_ctx.as_ref(), req.entity_type, req.entity_id, Some(req.action), @@ -368,7 +372,7 @@ impl crate::state::ServerState { entity_id: req.entity_id, }, action: req.action, - agent_ctx: req.agent_ctx, + agent_ctx: generation_agent_ctx.as_ref(), dispatch_idempotency_key: req.dispatch_idempotency_key, mode: req.mode, }; diff --git a/crates/temper-server/src/state/dispatch/wasm/local_tdata_host.rs b/crates/temper-server/src/state/dispatch/wasm/local_tdata_host.rs index b73165f45..fb91464fd 100644 --- a/crates/temper-server/src/state/dispatch/wasm/local_tdata_host.rs +++ b/crates/temper-server/src/state/dispatch/wasm/local_tdata_host.rs @@ -88,6 +88,7 @@ impl LocalTDataWasmHost { "POST" => crate::odata::handle_odata_post( State(self.state.clone()), None, + None, headers, Path(request.path), Query(request.query), diff --git a/crates/temper-server/src/state/entity_ops.rs b/crates/temper-server/src/state/entity_ops.rs index 633df47b1..45aea4810 100644 --- a/crates/temper-server/src/state/entity_ops.rs +++ b/crates/temper-server/src/state/entity_ops.rs @@ -1,21 +1,38 @@ //! Entity lifecycle methods for ServerState (spawn, query, delete, index). +mod actor_resolution; +#[cfg(test)] +#[path = "entity_ops/actor_resolution_test.rs"] +mod actor_resolution_test; +mod entity_loading; +mod field_updates; +mod generation_access; +mod key_activation; +mod key_index_coverage; +mod status_resolution; + use std::collections::BTreeMap; use std::sync::{Arc, OnceLock, RwLock}; use std::time::Instant; +use sha2::{Digest, Sha256}; use tracing::{Instrument, instrument}; use temper_observe::wide_event; -use temper_runtime::persistence::{EventMetadata, PersistenceEnvelope, PersistenceError}; +use temper_runtime::persistence::{ + EventMetadata, IndexReconciliation, PersistenceEnvelope, PersistenceError, SnapshotSourceFence, +}; use temper_runtime::scheduler::{sim_now, sim_uuid}; use temper_runtime::tenant::TenantId; use super::dispatch::retry; -use super::{ServerState, projection_backfill}; -use crate::entity_actor::{EntityActor, EntityEvent, EntityMsg, EntityResponse, EntityState}; +use super::{ServerState, SpecPublicationGuard, projection_backfill}; +use crate::entity_actor::{ + EntityActor, EntityEvent, EntityMsg, EntityPassivationSnapshot, EntityResponse, EntityState, +}; use crate::events::EntityStateChange; use crate::registry::{VerificationDetail, VerificationStatus}; +use crate::request_context::AgentContext; use crate::runtime_metrics; use crate::storage::DataOnlyCreateRecord; @@ -30,17 +47,6 @@ fn actor_idle_timeout_secs() -> i64 { }) } -fn is_deleted_envelope(event: &PersistenceEnvelope) -> bool { - if event.event_type == "Deleted" { - return true; - } - event - .payload - .get("action") - .and_then(serde_json::Value::as_str) - == Some("Deleted") -} - fn record_projection_update_started( tenant: &TenantId, entity_type: &str, @@ -154,7 +160,45 @@ pub(crate) struct AuthzResourceSnapshot { pub(crate) resource_attrs: BTreeMap, } +/// Fenced key-ownership work prepared before a registry publication. Callers +/// publish tables with [`Self::activation_epochs`], then finish the cutover to +/// evict old actors, publish coverage readiness, and reopen actor spawn. +#[derive(Debug)] +pub struct KeyContractActivationCutover { + /// Durable activation epoch assigned to each published or removed type. + pub activation_epochs: BTreeMap, + candidate_tables: Vec<(String, temper_jit::TransitionTable)>, + prepared_coverage: Vec, + affected_entity_types: std::collections::BTreeSet, +} + +impl KeyContractActivationCutover { + fn empty() -> Self { + Self { + activation_epochs: BTreeMap::new(), + candidate_tables: Vec::new(), + prepared_coverage: Vec::new(), + affected_entity_types: std::collections::BTreeSet::new(), + } + } +} + impl ServerState { + /// Arm a publication immediately before durable persistence. From this + /// point, cancellation or any publication error intentionally leaves the + /// tenant fail-closed until a serialized retry completes the cutover: a + /// failed commit acknowledgement cannot prove that no commit occurred. + pub fn arm_spec_publication( + &self, + publication_guard: &mut SpecPublicationGuard, + tenant: &TenantId, + intent_fingerprint: &str, + ) -> Result<(), String> { + publication_guard.arm(tenant, intent_fingerprint)?; + self.evict_tenant_actors(tenant); + Ok(()) + } + fn touch_actor_access(&self, actor_key: &str) { if let Ok(mut last_accessed) = self.last_accessed.write() { last_accessed.insert(actor_key.to_string(), sim_now()); @@ -455,32 +499,6 @@ impl ServerState { } } - /// Populate the durable query-plane projections for collection reads. - /// - /// Two-phase approach: - /// 1. **Snapshot pass** — cheap: deserialises snapshots for entities that have them. - /// 2. **Persistence replay pass** — reconstructs state directly from the event log. - /// - /// Runs once as a background task after startup. New entities created after - /// boot are indexed via `run_post_dispatch_effects` step 8. - #[instrument(skip_all, fields(otel.name = "entity.populate_field_index", tenant = %tenant))] - pub async fn populate_field_index_from_snapshots(&self, tenant: &TenantId) { - projection_backfill::populate_field_index_from_snapshots(self, tenant).await; - } - - /// Backfill `entity_key_index` for declared-key entity types (ADR-0153), so a - /// keyed read can authoritatively prove absence for pre-existing entities. - /// - /// Independent of [`Self::populate_field_index_from_snapshots`]: the declared - /// key is `K` (1–3) tiny rows per entity, far cheaper than the broad `S`-wide - /// field-index re-scan, so it is gated and scheduled on its own rather than - /// riding the expensive projection backfill. Runs once as a background task; - /// entities written after boot are keyed inline at write time. - #[instrument(skip_all, fields(otel.name = "entity.populate_key_index", tenant = %tenant))] - pub async fn populate_key_index_from_snapshots(&self, tenant: &TenantId) { - projection_backfill::populate_key_index_from_snapshots(self, tenant).await; - } - /// ADR-0155: backfill `entity_vector_index` for pre-existing entities of every /// vector-declaring type and record the watermark. Idempotent; entities written /// after boot maintain their vectors inline (co-commit) or write-behind. @@ -489,101 +507,6 @@ impl ServerState { projection_backfill::populate_vector_index_from_snapshots(self, tenant).await; } - /// Hydrate the per-tenant `entity_key_index` watermark cache once from the durable - /// watermark (ADR-0153). Safe to call repeatedly; conservative on any failure (leaves - /// the type uncovered → a keyed miss falls back to the scan, never a wrong "absent"). - async fn ensure_key_index_watermarks_loaded(&self, tenant: &TenantId) { - let already_loaded = self - .key_index_watermarks_loaded - .read() - .expect("key index watermarks-loaded lock poisoned") - .contains(tenant.as_str()); - if already_loaded { - return; - } - if let Some((store, _)) = self.event_journal() { - if let Ok(types) = store.key_index_backfilled_types(tenant.as_str()).await { - let mut cache = self - .key_index_backfilled - .write() - .expect("key index backfilled lock poisoned"); - for (et, key_set) in types { - cache.insert(format!("{tenant}:{et}"), key_set); - } - } - // Mark loaded even if the query errored: an error means "no authority - // yet", which is the safe scan-fallback state, and a completing - // backfill sets the cache entry directly via `mark_key_index_backfilled`. - self.key_index_watermarks_loaded - .write() - .expect("key index watermarks-loaded lock poisoned") - .insert(tenant.to_string()); - } - } - - /// The declared key-set the `entity_key_index` backfill covered for `(tenant, - /// entity_type)`, or `None` if the type was never backfilled. The value is the - /// sorted comma-joined declared key names (see [`declared_key_set_signature`]). - pub(crate) async fn key_index_backfill_covered_key_set( - &self, - tenant: &TenantId, - entity_type: &str, - ) -> Option { - self.ensure_key_index_watermarks_loaded(tenant).await; - self.key_index_backfilled - .read() - .expect("key index backfilled lock poisoned") - .get(&format!("{tenant}:{entity_type}")) - .cloned() - } - - /// Whether `entity_key_index` is complete for `(tenant, entity_type)` under the - /// CURRENT declared key-set — the ADR-0153 backfill watermark, made key-set aware - /// (ARN-68). True only when the covered key-set equals `current_key_set`, so a - /// keyed read MISS is authoritative absence only once EVERY currently-declared key - /// is backfilled; a newly-declared key reads as incomplete (scan-safe) until it is - /// re-keyed. Conservative on any failure (returns false → keyed miss falls back to - /// the scan, never a wrong "absent"). - pub(crate) async fn key_index_backfill_complete( - &self, - tenant: &TenantId, - entity_type: &str, - current_key_set: &str, - ) -> bool { - self.key_index_backfill_covered_key_set(tenant, entity_type) - .await - .as_deref() - == Some(current_key_set) - } - - /// Record (durably + in the read-path cache) that `entity_key_index` is complete - /// for `(tenant, entity_type)` covering exactly `key_set` (the sorted comma-joined - /// declared key names). Called by the backfill once it has keyed every existing - /// entity of the type, so subsequent keyed misses resolve to absence without - /// scanning (ADR-0153). Overwrites any stale key-set from an earlier declaration. - pub(crate) async fn mark_key_index_backfilled( - &self, - tenant: &TenantId, - entity_type: &str, - key_set: &str, - ) { - if let Some((store, _)) = self.event_journal() - && let Err(e) = store - .mark_key_index_backfilled(tenant.as_str(), entity_type, key_set) - .await - { - tracing::error!( - tenant = %tenant, entity_type, error = %e, - "failed to persist key-index backfill watermark" - ); - return; - } - self.key_index_backfilled - .write() - .expect("key index backfilled lock poisoned") - .insert(format!("{tenant}:{entity_type}"), key_set.to_string()); - } - /// Compare durable projection rows with authoritative state rebuilt by event replay. pub async fn verify_query_projection_replay_parity( &self, @@ -656,148 +579,6 @@ impl ServerState { } } - /// Get or spawn an entity actor (legacy single-tenant). - #[deprecated(note = "Use `get_or_spawn_tenant_actor` with explicit tenant")] - pub fn get_or_spawn_actor( - &self, - entity_type: &str, - entity_id: &str, - ) -> Option> { - self.get_or_spawn_tenant_actor(&TenantId::default(), entity_type, entity_id) - } - - /// Get or spawn an entity actor for a specific tenant. - pub fn get_or_spawn_tenant_actor( - &self, - tenant: &TenantId, - entity_type: &str, - entity_id: &str, - ) -> Option> { - self.get_or_spawn_tenant_actor_with_fields( - tenant, - entity_type, - entity_id, - serde_json::json!({}), - ) - } - - /// Get or spawn an entity actor with initial fields for a specific tenant. - #[instrument(skip_all, fields(otel.name = "entity.get_or_spawn_tenant_actor_with_fields", tenant = %tenant, entity_type, entity_id))] - pub fn get_or_spawn_tenant_actor_with_fields( - &self, - tenant: &TenantId, - entity_type: &str, - entity_id: &str, - initial_fields: serde_json::Value, - ) -> Option> { - let key = format!("{tenant}:{entity_type}:{entity_id}"); - - // Fast-path: check actor registry under read lock. - { - let registry = self.actor_registry.read().unwrap(); - if let Some(actor_ref) = registry.get(&key) { - self.touch_actor_access(&key); - return Some(actor_ref.clone()); - } - } - - // Look up live transition table reference: try SpecRegistry first, - // fall back to legacy map (wrapped in a fresh RwLock for compat). - let table = { - let reg = self.registry.read().unwrap(); - reg.get_table_live(tenant, entity_type) - } - .or_else(|| { - // Legacy single-tenant: wrap the static Arc in a - // new RwLock. Hot-swap doesn't apply to legacy mode, but the actor - // API is uniform. One clone per entity spawn (cheap). - self.transition_tables - .get(entity_type) - .map(|t| Arc::new(RwLock::new((**t).clone()))) - })?; - - // Build actor instance (spawn guarded below to avoid duplicate races). - // ADR-0048 sub-decision 5: every actor gets the shared idempotency - // cache so it can dedupe duplicate asks produced by retry storms. - let tenant_blob_store = self.blob_store_for_tenant(tenant).ok(); - let snapshot_queue = self - .snapshot_write_queue - .lock() - .ok() - .and_then(|slot| slot.clone()); - let actor = match self.event_journal() { - Some((store, backend)) => EntityActor::with_persistence( - entity_type, - entity_id, - table, - initial_fields, - store, - backend, - ) - .with_tenant(tenant.as_str()) - .with_snapshot_queue(snapshot_queue) - .with_idempotency_cache(self.idempotency_cache.clone()) - .with_blob_store(tenant_blob_store.clone()), - None => EntityActor::new(entity_type, entity_id, table, initial_fields) - .with_tenant(tenant.as_str()) - .with_idempotency_cache(self.idempotency_cache.clone()) - .with_blob_store(tenant_blob_store), - }; - - // Slow-path: atomically re-check and spawn under write lock. - // This prevents duplicate actors when concurrent requests race to create - // the same (tenant, entity_type, entity_id) key. - let actor_ref = { - let mut registry = self.actor_registry.write().unwrap(); - if let Some(existing) = registry.get(&key) { - return Some(existing.clone()); - } - let actor_ref = self.actor_system.spawn(actor, &key); - registry.insert(key.clone(), actor_ref.clone()); - actor_ref - }; - - // Track in entity index for collection queries - { - let index_key = format!("{tenant}:{entity_type}"); - let mut index = self.entity_index.write().unwrap(); - index - .entry(index_key) - .or_default() - .insert(entity_id.to_string()); - } - self.touch_actor_access(&key); - runtime_metrics::record_server_state_metrics(self); - - Some(actor_ref) - } - - /// Remove an entity from the index and actor registry. - #[instrument(skip_all, fields(otel.name = "entity.remove_entity", tenant = %tenant, entity_type, entity_id))] - pub fn remove_entity(&self, tenant: &TenantId, entity_type: &str, entity_id: &str) { - let actor_key = format!("{tenant}:{entity_type}:{entity_id}"); - - // Remove from actor registry - { - let mut registry = self.actor_registry.write().unwrap(); - registry.remove(&actor_key); - } - { - let mut last_accessed = self.last_accessed.write().unwrap(); - last_accessed.remove(&actor_key); - } - - // Remove from entity index - { - let index_key = format!("{tenant}:{entity_type}"); - let mut index = self.entity_index.write().unwrap(); - if let Some(ids) = index.get_mut(&index_key) { - ids.remove(entity_id); - } - } - runtime_metrics::record_server_state_metrics(self); - } - /// Stop and evict an entity actor plus its in-memory indexes. /// /// Used after an out-of-band durable append (for example, an atomic @@ -912,149 +693,6 @@ impl ServerState { } } - /// Get the current state of an entity actor (legacy single-tenant). - #[deprecated(note = "Use `get_tenant_entity_state` with explicit tenant")] - pub async fn get_entity_state( - &self, - entity_type: &str, - entity_id: &str, - ) -> Result { - self.get_tenant_entity_state(&TenantId::default(), entity_type, entity_id) - .await - } - - /// Get the current state of an entity actor for a specific tenant. - #[instrument(skip_all, fields(otel.name = "entity.get_tenant_entity_state", tenant = %tenant, entity_type, entity_id))] - pub async fn get_tenant_entity_state( - &self, - tenant: &TenantId, - entity_type: &str, - entity_id: &str, - ) -> Result { - let actor_ref = self - .get_or_spawn_tenant_actor(tenant, entity_type, entity_id) - .ok_or_else(|| { - format!("No transition table for tenant '{tenant}', entity type '{entity_type}'") - })?; - - // ADR-0048: retry transient ask failures (AskTimeout, MailboxFull) - // so a single slow actor reply does not surface as HTTP 500. - let policy = self.dispatch_retry_policy(); - retry::ask_with_backoff::<_, EntityResponse, _>(&actor_ref, || EntityMsg::GetState, &policy) - .await - .result - .map_err(|e| format!("Actor query failed: {e}")) - } - - /// Create a new entity with initial fields and return its state. - #[instrument(skip_all, fields(otel.name = "entity.get_or_create_tenant_entity", tenant = %tenant, entity_type, entity_id))] - pub async fn get_or_create_tenant_entity( - &self, - tenant: &TenantId, - entity_type: &str, - entity_id: &str, - initial_fields: serde_json::Value, - ) -> Result { - let actor_ref = self - .get_or_spawn_tenant_actor_with_fields(tenant, entity_type, entity_id, initial_fields) - .ok_or_else(|| { - format!("No transition table for tenant '{tenant}', entity type '{entity_type}'") - })?; - - let policy = self.dispatch_retry_policy(); - let response = retry::ask_with_backoff::<_, EntityResponse, _>( - &actor_ref, - || EntityMsg::GetState, - &policy, - ) - .await - .result - .map_err(|e| format!("Actor query failed: {e}"))?; - - // Broadcast entity creation event for SSE subscribers - let seq = self.next_entity_event_sequence(tenant.as_str(), entity_type, entity_id); - let change = EntityStateChange { - seq, - entity_type: entity_type.to_string(), - entity_id: entity_id.to_string(), - action: "Created".to_string(), - status: response.state.status.clone(), - tenant: tenant.to_string(), - agent_id: None, - session_id: None, - intent: None, - observation_metadata: None, - }; - self.record_entity_observe_event_with_seq( - tenant.as_str(), - entity_type, - entity_id, - seq, - "state_change", - serde_json::to_value(&change).unwrap_or_default(), - ); - let _ = self.event_tx.send(change); - - if let Some(query_plane) = self.query_plane_store() { - let status = response.state.status.clone(); - let fields = self.query_projection_fields(tenant, entity_type, &response.state.fields); - let projected_state = self.query_projection_state(&response.state); - let sequence_nr = response.state.sequence_nr; - let operation = "upsert"; - let source = "create"; - record_projection_update_started(tenant, entity_type, operation, source); - let projection_started_at = Instant::now(); // determinism-ok: production-only projection duration metric - if let Err(e) = query_plane - .upsert_projection( - tenant.as_str(), - entity_type, - entity_id, - &status, - &fields, - &projected_state, - sequence_nr, - ) - .await - { - record_projection_update_error( - tenant, - entity_type, - operation, - source, - projection_started_at, - ); - // Surface the projection failure instead of swallowing it. - // Previously this was a `warn` followed by `Ok(response)`, - // which produced HTTP 201 even when the entity wasn't - // discoverable via $filter — silently corrupting any caller - // that does write-then-read-back. Returning Err propagates - // up through the OData write handler to a 5xx so the client - // knows to retry. The event is already durable in the - // events table; on retry the actor's idempotent UPSERT into - // entity_catalog/entity_field_index will succeed (or fail - // again loudly). - tracing::error!( - error = %e, - tenant = %tenant, - entity_type = %entity_type, - entity_id = %entity_id, - "failed to update query projection during create" - ); - return Err(format!("query projection write failed during create: {e}")); - } - record_projection_update_success( - tenant, - entity_type, - operation, - source, - sequence_nr, - projection_started_at, - ); - } - - Ok(response) - } - /// Create a durable data-only entity without spawning an actor. /// /// This path is only eligible for transition tables with no rules. It @@ -1068,6 +706,9 @@ impl ServerState { entity_id: &str, initial_fields: serde_json::Value, ) -> Result, String> { + if self.key_contract_activation_gated(tenant, entity_type) { + return Ok(None); + } if !initial_fields.is_object() { return Ok(None); } @@ -1091,7 +732,7 @@ impl ServerState { .read() .expect("transition table lock poisoned") .clone(); - if !table.rules.is_empty() { + if !table.rules.is_empty() || !table.vectors.is_empty() { return Ok(None); } @@ -1156,6 +797,11 @@ impl ServerState { created_projection_state.sequence_nr = 1; created_projection_state.push_event_bounded(created.clone()); let projection_state = self.query_projection_state(&created_projection_state); + // The empty declared-key contract is authoritative too: it releases + // ownership rows from a previously keyed version of this entity type. + let reconcile_keys = true; + let key_set_signature = crate::key_index::declared_key_write_contract(&table); + let key_rows = crate::key_index::derive_entity_key_rows(&table.keys, &state.fields, true); if let Some(native_store) = self.data_only_create_store() { let operation = "native_create"; let source = "data_only_create_fast_path"; @@ -1177,6 +823,9 @@ impl ServerState { fields: &projection_fields, state: &projection_state, event: &envelope, + key_rows: &key_rows, + reconcile_keys, + key_set_signature: &key_set_signature, }) .instrument(native_span) .await @@ -1192,7 +841,10 @@ impl ServerState { projection_started_at, ); } - Err(PersistenceError::ConcurrencyViolation { .. }) => { + Err( + PersistenceError::ConcurrencyViolation { .. } + | PersistenceError::SnapshotGenerationChanged, + ) => { record_projection_update_error( tenant, entity_type, @@ -1225,18 +877,37 @@ impl ServerState { } else { let append_started_at = Instant::now(); // determinism-ok: production-only append wait metric let append_result = store - .append(&persistence_id, state.sequence_nr, &[envelope]) + .append_with_index_rows( + &persistence_id, + state.sequence_nr, + &[envelope], + &key_rows, + &[], + IndexReconciliation { + keys: reconcile_keys, + key_set_signature: Some(key_set_signature), + vectors: false, + snapshot_source: SnapshotSourceFence::Absent, + }, + ) .await; runtime_metrics::record_event_store_append_wait( backend.as_str(), - "append", + if reconcile_keys { + "append_with_keys" + } else { + "append" + }, append_started_at.elapsed(), ); match append_result { Ok(new_seq) => { state.sequence_nr = new_seq; } - Err(PersistenceError::ConcurrencyViolation { .. }) => { + Err( + PersistenceError::ConcurrencyViolation { .. } + | PersistenceError::SnapshotGenerationChanged, + ) => { return Ok(None); } Err(e) => { @@ -1335,91 +1006,6 @@ impl ServerState { })) } - /// Update fields on an existing entity. - #[instrument(skip_all, fields(otel.name = "entity.update_tenant_entity_fields", tenant = %tenant, entity_type, entity_id))] - pub async fn update_tenant_entity_fields( - &self, - tenant: &TenantId, - entity_type: &str, - entity_id: &str, - fields: serde_json::Value, - replace: bool, - ) -> Result { - let actor_ref = self - .get_or_spawn_tenant_actor(tenant, entity_type, entity_id) - .ok_or_else(|| { - format!("No transition table for tenant '{tenant}', entity type '{entity_type}'") - })?; - - let policy = self.dispatch_retry_policy(); - let fields_for_retry = fields; - let response = retry::ask_with_backoff::<_, EntityResponse, _>( - &actor_ref, - || EntityMsg::UpdateFields { - fields: fields_for_retry.clone(), - replace, - }, - &policy, - ) - .await - .result - .map_err(|e| format!("Actor update failed: {e}"))?; - - if response.success - && let Some(query_plane) = self.query_plane_store() - { - let status = response.state.status.clone(); - let fields = self.query_projection_fields(tenant, entity_type, &response.state.fields); - let projected_state = self.query_projection_state(&response.state); - let sequence_nr = response.state.sequence_nr; - let operation = "upsert"; - let source = "field_update"; - record_projection_update_started(tenant, entity_type, operation, source); - let projection_started_at = Instant::now(); // determinism-ok: production-only projection duration metric - if let Err(e) = query_plane - .upsert_projection( - tenant.as_str(), - entity_type, - entity_id, - &status, - &fields, - &projected_state, - sequence_nr, - ) - .await - { - record_projection_update_error( - tenant, - entity_type, - operation, - source, - projection_started_at, - ); - // Same reasoning as create: don't ack a write that won't be - // visible via $filter. Propagate so OData returns 5xx and - // clients can retry against the idempotent upsert. - tracing::error!( - error = %e, - tenant = %tenant, - entity_type = %entity_type, - entity_id = %entity_id, - "failed to update query projection during field update" - ); - return Err(format!("query projection write failed during update: {e}")); - } - record_projection_update_success( - tenant, - entity_type, - operation, - source, - sequence_nr, - projection_started_at, - ); - } - - Ok(response) - } - /// Delete an entity. #[instrument(skip_all, fields(otel.name = "entity.delete_tenant_entity", tenant = %tenant, entity_type, entity_id))] pub async fn delete_tenant_entity( @@ -1503,75 +1089,6 @@ impl ServerState { .is_some_and(|ids| ids.contains(entity_id)) } - /// Ensure an entity is present in memory by lazily hydrating from the - /// event store when needed. - #[instrument(skip_all, fields(otel.name = "entity.ensure_entity_loaded", tenant = %tenant, entity_type, entity_id))] - pub async fn ensure_entity_loaded( - &self, - tenant: &TenantId, - entity_type: &str, - entity_id: &str, - ) -> bool { - let persistence_id = format!("{tenant}:{entity_type}:{entity_id}"); - let journal = self.event_journal(); - - if self.entity_exists(tenant, entity_type, entity_id) { - let Some((store, _backend)) = journal.as_ref() else { - return true; - }; - - let events = match store.read_events(&persistence_id, 0).await { - Ok(events) if !events.is_empty() => events, - _ => return true, - }; - - if events.last().is_some_and(is_deleted_envelope) { - self.remove_entity(tenant, entity_type, entity_id); - return false; - } - - return true; - } - - let Some((store, _backend)) = journal.as_ref() else { - return false; - }; - - let events = match store.read_events(&persistence_id, 0).await { - Ok(events) if !events.is_empty() => events, - _ => return false, - }; - - if events.last().is_some_and(is_deleted_envelope) { - self.remove_entity(tenant, entity_type, entity_id); - return false; - } - - let Some(actor_ref) = self.get_or_spawn_tenant_actor(tenant, entity_type, entity_id) else { - return false; - }; - - let policy = self.dispatch_retry_policy(); - let outcome = retry::ask_with_backoff::<_, EntityResponse, _>( - &actor_ref, - || EntityMsg::GetState, - &policy, - ) - .await; - match outcome.result { - Ok(response) if response.state.status == "Deleted" => { - let _ = actor_ref.stop(); - self.remove_entity(tenant, entity_type, entity_id); - false - } - Ok(_) => true, - Err(_) => { - self.remove_entity(tenant, entity_type, entity_id); - false - } - } - } - /// List entity IDs for a type, guaranteeing completeness against the /// durable event store. /// @@ -1641,11 +1158,19 @@ impl ServerState { let policy = self.dispatch_retry_policy(); let journal = self.event_journal(); for (actor_key, actor_ref) in candidates { + if temper_runtime::tenant::parse_persistence_id_parts(&actor_key) + .ok() + .is_some_and(|(tenant, entity_type, _)| { + self.key_contract_activation_gated(&TenantId::new(tenant), entity_type) + }) + { + continue; + } // ADR-0048: retry transient failures so passivation is not skipped // by a single AskTimeout under load. - let snapshot_outcome = retry::ask_with_backoff::<_, EntityResponse, _>( + let snapshot_outcome = retry::ask_with_backoff::<_, EntityPassivationSnapshot, _>( &actor_ref, - || EntityMsg::GetState, + || EntityMsg::GetPassivationSnapshot, &policy, ) .await; @@ -1653,29 +1178,58 @@ impl ServerState { && let Ok(response) = snapshot_outcome.result && response.state.sequence_nr > 0 { - // Snapshot excludes bounded in-memory recent event history. - let mut snapshot_value = match serde_json::to_value(&response.state) { - Ok(v) => v, - Err(e) => { - tracing::warn!(actor_key = %actor_key, error = %e, "failed to encode snapshot value"); - serde_json::Value::Null + let journal_provenance = match store.journal_boundary(&actor_key).await { + Ok(boundary) if boundary.latest_sequence == 0 => Some(None), + Ok(boundary) if boundary.latest_sequence == response.state.sequence_nr => { + Some(Some(boundary.latest_sequence)) + } + Ok(boundary) => { + tracing::warn!( + actor_key = %actor_key, + actor_sequence = response.state.sequence_nr, + journal_sequence = boundary.latest_sequence, + "skipping passivation snapshot because actor and journal differ" + ); + None + } + Err(error) => { + tracing::warn!( + actor_key = %actor_key, + error = %error, + "skipping passivation snapshot because journal provenance is unavailable" + ); + None } }; - if let Some(obj) = snapshot_value.as_object_mut() { - obj.remove("events"); - } - if !snapshot_value.is_null() - && let Ok(snapshot_bytes) = serde_json::to_vec(&snapshot_value) - && let Err(e) = store - .save_snapshot(&actor_key, response.state.sequence_nr, &snapshot_bytes) - .await - { - tracing::warn!( - actor_key = %actor_key, - seq = response.state.sequence_nr, - error = %e, - "failed to save snapshot during passivation" - ); + if let Some(journal_sequence) = journal_provenance { + match EntityActor::serialize_snapshot_state(&response.state, journal_sequence) { + Ok(snapshot_bytes) => { + if let Err(e) = store + .save_snapshot_if_source( + &actor_key, + response.state.sequence_nr, + &snapshot_bytes, + &response.snapshot_source, + Some(&response.key_contract), + ) + .await + { + tracing::warn!( + actor_key = %actor_key, + seq = response.state.sequence_nr, + error = %e, + "failed to save snapshot during passivation" + ); + } + } + Err(e) => { + tracing::warn!( + actor_key = %actor_key, + error = %e, + "failed to encode snapshot during passivation" + ); + } + } } } @@ -1810,39 +1364,5 @@ impl ServerState { } } -/// Resolve the current status of an entity. -/// -/// Fast path: check the `entity_state_cache` (populated on every successful dispatch). -/// Slow path: fall back to `get_tenant_entity_state()` (async actor ask) and backfill cache. -impl ServerState { - #[instrument(skip_all, fields(otel.name = "entity.resolve_entity_status", tenant = %tenant, entity_type, entity_id))] - pub async fn resolve_entity_status( - &self, - tenant: &TenantId, - entity_type: &str, - entity_id: &str, - ) -> Option { - // Fast path: check cache (LruCache::get requires &mut, so use Mutex). - let cache_key = format!("{tenant}:{entity_type}:{entity_id}"); - if let Ok(mut cache) = self.entity_state_cache.lock() - && let Some((status, _timestamp)) = cache.get(&cache_key) - { - return Some(status.clone()); - } - - // Slow path: actor ask + backfill - if let Ok(response) = self - .get_tenant_entity_state(tenant, entity_type, entity_id) - .await - { - let status = response.state.status.clone(); - self.cache_entity_status(cache_key, status.clone()); - Some(status) - } else { - None - } - } -} - use temper_authz::{AuthzDecision, AuthzDenial, SecurityContext}; use temper_runtime::actor::ActorRef; diff --git a/crates/temper-server/src/state/entity_ops/actor_resolution.rs b/crates/temper-server/src/state/entity_ops/actor_resolution.rs new file mode 100644 index 000000000..fdba23405 --- /dev/null +++ b/crates/temper-server/src/state/entity_ops/actor_resolution.rs @@ -0,0 +1,326 @@ +//! Generation-aware actor resolution and eviction. + +use super::*; + +impl ServerState { + /// Get or spawn an entity actor (legacy single-tenant). + #[deprecated(note = "Use `get_or_spawn_tenant_actor` with explicit tenant")] + pub fn get_or_spawn_actor( + &self, + entity_type: &str, + entity_id: &str, + ) -> Option> { + self.get_or_spawn_tenant_actor(&TenantId::default(), entity_type, entity_id) + } + + /// Get or spawn an entity actor for a specific tenant. + pub fn get_or_spawn_tenant_actor( + &self, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + ) -> Option> { + self.get_or_spawn_tenant_actor_with_fields_in_context( + tenant, + entity_type, + entity_id, + serde_json::json!({}), + None, + ) + } + + /// Resolve an actor while preserving an already-admitted tenant generation. + pub(crate) fn get_or_spawn_tenant_actor_in_generation( + &self, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + agent_ctx: &AgentContext, + ) -> Option> { + self.get_or_spawn_tenant_actor_with_fields_in_context( + tenant, + entity_type, + entity_id, + serde_json::json!({}), + Some(agent_ctx), + ) + } + + /// Get or spawn an entity actor with initial fields for a specific tenant. + #[instrument(skip_all, fields(otel.name = "entity.get_or_spawn_tenant_actor_with_fields", tenant = %tenant, entity_type, entity_id))] + pub fn get_or_spawn_tenant_actor_with_fields( + &self, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + initial_fields: serde_json::Value, + ) -> Option> { + self.get_or_spawn_tenant_actor_with_fields_in_context( + tenant, + entity_type, + entity_id, + initial_fields, + None, + ) + } + + pub(super) fn get_or_spawn_tenant_actor_with_fields_in_context( + &self, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + initial_fields: serde_json::Value, + agent_ctx: Option<&AgentContext>, + ) -> Option> { + let key = format!("{tenant}:{entity_type}:{entity_id}"); + if self.actor_resolution_gated(tenant, entity_type, agent_ctx) { + tracing::debug!( + tenant = %tenant, + entity_type, + entity_id, + "actor spawn deferred while key contract activation is not ready" + ); + return None; + } + + // Fast-path: check actor registry under read lock. + { + let registry = self.actor_registry.read().unwrap(); + if let Some(actor_ref) = registry.get(&key) { + self.touch_actor_access(&key); + return Some(actor_ref.clone()); + } + } + + // Look up live transition table reference: try SpecRegistry first, + // fall back to legacy map (wrapped in a fresh RwLock for compat). + let table = { + let reg = self.registry.read().unwrap(); + reg.get_table_live(tenant, entity_type) + } + .or_else(|| { + // Legacy single-tenant: wrap the static Arc in a + // new RwLock. Hot-swap doesn't apply to legacy mode, but the actor + // API is uniform. One clone per entity spawn (cheap). + self.transition_tables + .get(entity_type) + .map(|t| Arc::new(RwLock::new((**t).clone()))) + })?; + + // Build actor instance (spawn guarded below to avoid duplicate races). + // ADR-0048 sub-decision 5: every actor gets the shared idempotency + // cache so it can dedupe duplicate asks produced by retry storms. + let tenant_blob_store = self.blob_store_for_tenant(tenant).ok(); + let snapshot_queue = self + .snapshot_write_queue + .lock() + .ok() + .and_then(|slot| slot.clone()); + let actor = match self.event_journal() { + Some((store, backend)) => EntityActor::with_persistence( + entity_type, + entity_id, + table, + initial_fields, + store, + backend, + ) + .with_tenant(tenant.as_str()) + .with_snapshot_queue(snapshot_queue) + .with_idempotency_cache(self.idempotency_cache.clone()) + .with_blob_store(tenant_blob_store.clone()), + None => EntityActor::new(entity_type, entity_id, table, initial_fields) + .with_tenant(tenant.as_str()) + .with_idempotency_cache(self.idempotency_cache.clone()) + .with_blob_store(tenant_blob_store), + }; + + // Slow-path: atomically re-check and spawn under write lock. + // This prevents duplicate actors when concurrent requests race to create + // the same (tenant, entity_type, entity_id) key. + let actor_ref = { + let mut registry = self.actor_registry.write().unwrap(); + if let Some(existing) = registry.get(&key) { + return Some(existing.clone()); + } + // Linearize actor insertion against publication arm. If this + // request observed the old registry before the gate closed, it + // must not insert that old-table actor after arm's first eviction. + if self.actor_resolution_gated(tenant, entity_type, agent_ctx) { + return None; + } + let actor_ref = self.actor_system.spawn(actor, &key); + registry.insert(key.clone(), actor_ref.clone()); + actor_ref + }; + + // Track in entity index for collection queries + { + let index_key = format!("{tenant}:{entity_type}"); + let mut index = self.entity_index.write().unwrap(); + index + .entry(index_key) + .or_default() + .insert(entity_id.to_string()); + } + self.touch_actor_access(&key); + runtime_metrics::record_server_state_metrics(self); + + Some(actor_ref) + } + + pub(super) fn actor_resolution_gated( + &self, + tenant: &TenantId, + entity_type: &str, + agent_ctx: Option<&AgentContext>, + ) -> bool { + if self + .activating_key_contracts + .read() + .expect("activating key contracts lock poisoned") + .contains(&(tenant.as_str().to_string(), entity_type.to_string())) + { + return true; + } + if !self.spec_publication_gated(tenant) { + return false; + } + + !agent_ctx + .and_then(|ctx| ctx.tenant_generation_lease.as_ref()) + .is_some_and(|lease| { + lease.belongs_to(tenant) + && lease.captured_generation() == self.tenant_generation_version(tenant) + && (lease.is_held_for(tenant) || lease.is_publication_owned_for(tenant)) + }) + } + + /// Evict a freshly spawned actor when its first dispatch failed before any + /// durable history was admitted. + /// + /// A failed actor initialization can stop before writing its bootstrap + /// event. The spawn path has already published the ID to `entity_index` at + /// that point, so leaving it behind would advertise a phantom entity and + /// violate index/store agreement. A durable read is the authority: an + /// ambiguous or non-empty journal is preserved for recovery. + pub(crate) async fn discard_uncommitted_spawn_after_dispatch_failure( + &self, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + expected_actor_id: &temper_runtime::actor::ActorId, + ) { + let Some((store, _backend)) = self.event_journal() else { + self.stop_and_remove_actor_incarnation( + tenant, + entity_type, + entity_id, + expected_actor_id, + true, + ); + return; + }; + let persistence_id = format!("{tenant}:{entity_type}:{entity_id}"); + match store.read_events(&persistence_id, 0).await { + Ok(events) if events.is_empty() => { + self.stop_and_remove_actor_incarnation( + tenant, + entity_type, + entity_id, + expected_actor_id, + true, + ); + } + Ok(_) => { + self.stop_and_remove_actor_incarnation( + tenant, + entity_type, + entity_id, + expected_actor_id, + false, + ); + } + Err(error) => { + tracing::warn!( + tenant = %tenant, + entity_type, + entity_id, + error = %error, + "preserving entity index because failed actor durability is ambiguous" + ); + self.stop_and_remove_actor_incarnation( + tenant, + entity_type, + entity_id, + expected_actor_id, + false, + ); + } + } + } + + fn stop_and_remove_actor_incarnation( + &self, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + expected_actor_id: &temper_runtime::actor::ActorId, + remove_from_index: bool, + ) { + let actor_key = format!("{tenant}:{entity_type}:{entity_id}"); + let Ok(mut registry) = self.actor_registry.write() else { + return; + }; + let Some(current) = registry.get(&actor_key) else { + return; + }; + if current.id() != expected_actor_id { + return; + } + let removed = registry.remove(&actor_key); + if let Some(actor_ref) = removed { + let _ = actor_ref.stop(); + } + // Keep the registry write lock until the correlated maps are cleared, + // so a replacement incarnation cannot publish entries between removal + // and cleanup. + if let Ok(mut last_accessed) = self.last_accessed.write() { + last_accessed.remove(&actor_key); + } + if remove_from_index && let Ok(mut index) = self.entity_index.write() { + let index_key = format!("{tenant}:{entity_type}"); + if let Some(ids) = index.get_mut(&index_key) { + ids.remove(entity_id); + } + } + drop(registry); + runtime_metrics::record_server_state_metrics(self); + } + + /// Remove an entity from the index and actor registry. + #[instrument(skip_all, fields(otel.name = "entity.remove_entity", tenant = %tenant, entity_type, entity_id))] + pub fn remove_entity(&self, tenant: &TenantId, entity_type: &str, entity_id: &str) { + let actor_key = format!("{tenant}:{entity_type}:{entity_id}"); + + // Remove from actor registry + { + let mut registry = self.actor_registry.write().unwrap(); + registry.remove(&actor_key); + } + { + let mut last_accessed = self.last_accessed.write().unwrap(); + last_accessed.remove(&actor_key); + } + + // Remove from entity index + { + let index_key = format!("{tenant}:{entity_type}"); + let mut index = self.entity_index.write().unwrap(); + if let Some(ids) = index.get_mut(&index_key) { + ids.remove(entity_id); + } + } + runtime_metrics::record_server_state_metrics(self); + } +} diff --git a/crates/temper-server/src/state/entity_ops/actor_resolution_test.rs b/crates/temper-server/src/state/entity_ops/actor_resolution_test.rs new file mode 100644 index 000000000..05ea9ae7c --- /dev/null +++ b/crates/temper-server/src/state/entity_ops/actor_resolution_test.rs @@ -0,0 +1,244 @@ +use std::sync::{Arc, RwLock}; + +use temper_jit::table::TransitionTable; +use temper_runtime::ActorSystem; +use temper_runtime::actor::ActorRef; +use temper_runtime::persistence::{EventMetadata, EventStore, PersistenceEnvelope}; +use temper_runtime::scheduler::{install_deterministic_context, sim_now, sim_uuid}; +use temper_runtime::tenant::TenantId; +use temper_spec::csdl::parse_csdl; +use temper_store_sim::SimEventStore; + +use crate::entity_actor::{EntityActor, EntityMsg}; +use crate::storage::StorageStack; + +use super::ServerState; + +const CSDL_XML: &str = include_str!("../../../../../test-fixtures/specs/model.csdl.xml"); +const ORDER_IOA: &str = include_str!("../../../../../test-fixtures/specs/order.ioa.toml"); + +fn test_state(seed: u64) -> (ServerState, SimEventStore) { + let csdl = parse_csdl(CSDL_XML).expect("CSDL parses"); + let mut state = ServerState::new( + ActorSystem::new("actor-resolution-cleanup"), + csdl, + CSDL_XML.to_string(), + ); + let store = SimEventStore::no_faults(seed); + state.set_storage_stack(StorageStack::from_sim(store.clone(), None)); + (state, store) +} + +fn insert_actor(state: &ServerState, tenant: &TenantId, entity_id: &str) -> ActorRef { + let actor_key = format!("{tenant}:Order:{entity_id}"); + let actor_ref = state.actor_system.spawn( + EntityActor::new( + "Order", + entity_id, + Arc::new(RwLock::new(TransitionTable::from_ioa_source(ORDER_IOA))), + serde_json::json!({}), + ) + .with_tenant(tenant.as_str()), + &actor_key, + ); + state + .actor_registry + .write() + .expect("actor registry lock") + .insert(actor_key.clone(), actor_ref.clone()); + state + .last_accessed + .write() + .expect("actor access lock") + .insert(actor_key, sim_now()); + state + .entity_index + .write() + .expect("entity index lock") + .entry(format!("{tenant}:Order")) + .or_default() + .insert(entity_id.to_string()); + actor_ref +} + +fn assert_actor_is_indexed( + state: &ServerState, + tenant: &TenantId, + entity_id: &str, + expected: &ActorRef, +) { + let actor_key = format!("{tenant}:Order:{entity_id}"); + assert_eq!( + state + .actor_registry + .read() + .expect("actor registry lock") + .get(&actor_key) + .map(ActorRef::id), + Some(expected.id()) + ); + assert!( + state + .entity_index + .read() + .expect("entity index lock") + .get(&format!("{tenant}:Order")) + .is_some_and(|ids| ids.contains(entity_id)) + ); +} + +fn assert_terminal_actor_evicted_but_indexed( + state: &ServerState, + tenant: &TenantId, + entity_id: &str, +) { + let actor_key = format!("{tenant}:Order:{entity_id}"); + assert!( + !state + .actor_registry + .read() + .expect("actor registry lock") + .contains_key(&actor_key) + ); + assert!( + !state + .last_accessed + .read() + .expect("actor access lock") + .contains_key(&actor_key) + ); + assert!( + state + .entity_index + .read() + .expect("entity index lock") + .get(&format!("{tenant}:Order")) + .is_some_and(|ids| ids.contains(entity_id)) + ); +} + +#[tokio::test] +async fn failed_cleanup_removes_only_the_empty_actor_incarnation() { + let (_guard, _clock, _ids) = install_deterministic_context(237); + let (state, _store) = test_state(237); + let tenant = TenantId::default(); + let entity_id = "empty-history"; + let actor_key = format!("{tenant}:Order:{entity_id}"); + let actor_ref = insert_actor(&state, &tenant, entity_id); + + state + .discard_uncommitted_spawn_after_dispatch_failure( + &tenant, + "Order", + entity_id, + actor_ref.id(), + ) + .await; + + assert!( + !state + .actor_registry + .read() + .expect("actor registry lock") + .contains_key(&actor_key) + ); + assert!( + !state + .last_accessed + .read() + .expect("actor access lock") + .contains_key(&actor_key) + ); + assert!( + !state + .entity_index + .read() + .expect("entity index lock") + .get(&format!("{tenant}:Order")) + .is_some_and(|ids| ids.contains(entity_id)) + ); +} + +#[tokio::test] +async fn failed_cleanup_preserves_a_replacement_actor_incarnation() { + let (_guard, _clock, _ids) = install_deterministic_context(238); + let (state, _store) = test_state(238); + let tenant = TenantId::default(); + let entity_id = "replacement-incarnation"; + let original = insert_actor(&state, &tenant, entity_id); + let replacement = insert_actor(&state, &tenant, entity_id); + assert_ne!(original.id(), replacement.id()); + + state + .discard_uncommitted_spawn_after_dispatch_failure( + &tenant, + "Order", + entity_id, + original.id(), + ) + .await; + + assert_actor_is_indexed(&state, &tenant, entity_id, &replacement); +} + +#[tokio::test] +async fn failed_cleanup_evicts_terminal_actor_but_preserves_index_when_history_read_is_ambiguous() { + let (_guard, _clock, _ids) = install_deterministic_context(239); + let (state, store) = test_state(239); + let tenant = TenantId::default(); + let entity_id = "ambiguous-history"; + let actor_ref = insert_actor(&state, &tenant, entity_id); + store.fail_next_reads(&format!("{tenant}:Order:{entity_id}"), 1); + + state + .discard_uncommitted_spawn_after_dispatch_failure( + &tenant, + "Order", + entity_id, + actor_ref.id(), + ) + .await; + + assert_terminal_actor_evicted_but_indexed(&state, &tenant, entity_id); +} + +#[tokio::test] +async fn failed_cleanup_evicts_terminal_actor_but_preserves_index_with_durable_history() { + let (_guard, _clock, _ids) = install_deterministic_context(240); + let (state, store) = test_state(240); + let tenant = TenantId::default(); + let entity_id = "durable-history"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let actor_id = persistence_id.clone(); + let actor_ref = insert_actor(&state, &tenant, entity_id); + store + .append( + &persistence_id, + 0, + &[PersistenceEnvelope { + sequence_nr: 1, + event_type: "Created".to_string(), + payload: serde_json::json!({}), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: sim_now(), + actor_id, + }, + }], + ) + .await + .expect("seed durable history"); + + state + .discard_uncommitted_spawn_after_dispatch_failure( + &tenant, + "Order", + entity_id, + actor_ref.id(), + ) + .await; + + assert_terminal_actor_evicted_but_indexed(&state, &tenant, entity_id); +} diff --git a/crates/temper-server/src/state/entity_ops/entity_loading.rs b/crates/temper-server/src/state/entity_ops/entity_loading.rs new file mode 100644 index 000000000..545b5ae46 --- /dev/null +++ b/crates/temper-server/src/state/entity_ops/entity_loading.rs @@ -0,0 +1,140 @@ +//! Generation-aware lazy entity hydration. + +use super::*; + +impl ServerState { + /// Ensure an entity is present in memory by lazily hydrating from the + /// event store when needed. + #[instrument(skip_all, fields(otel.name = "entity.ensure_entity_loaded", tenant = %tenant, entity_type, entity_id))] + pub async fn ensure_entity_loaded( + &self, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + ) -> bool { + self.ensure_entity_loaded_with_context(tenant, entity_type, entity_id, None) + .await + } + + /// Hydrate an entity inside an already-admitted tenant generation. + pub async fn ensure_entity_loaded_in_generation( + &self, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + agent_ctx: &AgentContext, + ) -> bool { + self.ensure_entity_loaded_with_context(tenant, entity_type, entity_id, Some(agent_ctx)) + .await + } + + pub(super) async fn ensure_entity_loaded_with_context( + &self, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + agent_ctx: Option<&AgentContext>, + ) -> bool { + let persistence_id = format!("{tenant}:{entity_type}:{entity_id}"); + let Some((store, _backend)) = self.event_journal() else { + return self.entity_exists(tenant, entity_type, entity_id); + }; + + // Snapshot replacement and the first materializing journal append are + // stream-fenced, but can race this read. Retry a bounded number of exact + // source captures instead of accepting an actor hydrated from a source + // generation that changed underneath the existence check. + for _attempt in 0..3 { + let resident = self.entity_exists(tenant, entity_type, entity_id); + let boundary = match store.journal_boundary(&persistence_id).await { + Ok(boundary) => boundary, + Err(_) => return resident, + }; + if boundary.first_terminal_sequence.is_some() { + self.remove_entity(tenant, entity_type, entity_id); + return false; + } + if resident && boundary.latest_sequence > 0 { + return true; + } + + let captured_snapshot = if boundary.latest_sequence == 0 { + match store.load_snapshot(&persistence_id).await { + Ok(Some(snapshot)) => Some(snapshot), + Ok(None) => return resident, + Err(_) => return resident, + } + } else { + None + }; + + let Some(actor_ref) = self.get_or_spawn_tenant_actor_with_fields_in_context( + tenant, + entity_type, + entity_id, + serde_json::json!({}), + agent_ctx, + ) else { + return false; + }; + let policy = self.dispatch_retry_policy(); + if let Some(captured_snapshot) = captured_snapshot { + let outcome = retry::ask_with_backoff::<_, EntityPassivationSnapshot, _>( + &actor_ref, + || EntityMsg::GetPassivationSnapshot, + &policy, + ) + .await; + let response = match outcome.result { + Ok(response) => response, + Err(_) => { + self.stop_and_remove_entity(tenant, entity_type, entity_id); + return false; + } + }; + if response.state.status == "Deleted" { + self.stop_and_remove_entity(tenant, entity_type, entity_id); + return false; + } + let actor_used_captured_source = response.snapshot_source + == (SnapshotSourceFence::Exact { + sequence_nr: captured_snapshot.0, + state: captured_snapshot.1.clone(), + }); + let source_still_exact = matches!( + ( + store.journal_boundary(&persistence_id).await, + store.load_snapshot(&persistence_id).await, + ), + (Ok(boundary), Ok(Some(current_snapshot))) + if boundary.latest_sequence == 0 && current_snapshot == captured_snapshot + ); + if !actor_used_captured_source || !source_still_exact { + self.stop_and_remove_entity(tenant, entity_type, entity_id); + continue; + } + } else { + let outcome = retry::ask_with_backoff::<_, EntityResponse, _>( + &actor_ref, + || EntityMsg::GetState, + &policy, + ) + .await; + match outcome.result { + Ok(response) if response.state.status == "Deleted" => { + self.stop_and_remove_entity(tenant, entity_type, entity_id); + return false; + } + Ok(_) => {} + Err(_) => { + self.stop_and_remove_entity(tenant, entity_type, entity_id); + return false; + } + } + } + return true; + } + + false + } +} diff --git a/crates/temper-server/src/state/entity_ops/field_updates.rs b/crates/temper-server/src/state/entity_ops/field_updates.rs new file mode 100644 index 000000000..f19c11b2b --- /dev/null +++ b/crates/temper-server/src/state/entity_ops/field_updates.rs @@ -0,0 +1,158 @@ +//! Generation-aware entity field updates. + +use super::*; + +impl ServerState { + /// Update fields on an existing entity. + #[instrument(skip_all, fields(otel.name = "entity.update_tenant_entity_fields", tenant = %tenant, entity_type, entity_id))] + pub async fn update_tenant_entity_fields( + &self, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + fields: serde_json::Value, + replace: bool, + idempotency_key: Option, + ) -> Result { + self.update_tenant_entity_fields_with_context( + tenant, + entity_type, + entity_id, + fields, + replace, + idempotency_key, + None, + ) + .await + } + + /// Update entity fields inside an already-admitted tenant generation. + #[expect( + clippy::too_many_arguments, + reason = "generation-aware update preserves the public field-update contract" + )] + pub async fn update_tenant_entity_fields_in_generation( + &self, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + fields: serde_json::Value, + replace: bool, + idempotency_key: Option, + agent_ctx: &AgentContext, + ) -> Result { + self.update_tenant_entity_fields_with_context( + tenant, + entity_type, + entity_id, + fields, + replace, + idempotency_key, + Some(agent_ctx), + ) + .await + } + + #[expect( + clippy::too_many_arguments, + reason = "internal field update carries the optional captured agent context" + )] + pub(super) async fn update_tenant_entity_fields_with_context( + &self, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + fields: serde_json::Value, + replace: bool, + idempotency_key: Option, + agent_ctx: Option<&AgentContext>, + ) -> Result { + let actor_ref = self + .get_or_spawn_tenant_actor_with_fields_in_context( + tenant, + entity_type, + entity_id, + serde_json::json!({}), + agent_ctx, + ) + .ok_or_else(|| { + format!("No transition table for tenant '{tenant}', entity type '{entity_type}'") + })?; + + let policy = self.dispatch_retry_policy(); + let fields_for_retry = fields; + let idempotency_key = + idempotency_key.unwrap_or_else(|| format!("field-update:{}", sim_uuid())); + let response = retry::ask_with_backoff::<_, EntityResponse, _>( + &actor_ref, + || EntityMsg::UpdateFields { + fields: fields_for_retry.clone(), + replace, + idempotency_key: idempotency_key.clone(), + }, + &policy, + ) + .await + .result + .map_err(|e| format!("Actor update failed: {e}"))?; + + if !response.success { + return Err(response + .error + .clone() + .unwrap_or_else(|| "field update was rejected".to_string())); + } + + if let Some(query_plane) = self.query_plane_store() { + let status = response.state.status.clone(); + let fields = self.query_projection_fields(tenant, entity_type, &response.state.fields); + let projected_state = self.query_projection_state(&response.state); + let sequence_nr = response.state.sequence_nr; + let operation = "upsert"; + let source = "field_update"; + record_projection_update_started(tenant, entity_type, operation, source); + let projection_started_at = Instant::now(); // determinism-ok: production-only projection duration metric + if let Err(e) = query_plane + .upsert_projection( + tenant.as_str(), + entity_type, + entity_id, + &status, + &fields, + &projected_state, + sequence_nr, + ) + .await + { + record_projection_update_error( + tenant, + entity_type, + operation, + source, + projection_started_at, + ); + // Same reasoning as create: don't ack a write that won't be + // visible via $filter. Propagate so OData returns 5xx and + // clients can retry against the idempotent upsert. + tracing::error!( + error = %e, + tenant = %tenant, + entity_type = %entity_type, + entity_id = %entity_id, + "failed to update query projection during field update" + ); + return Err(format!("query projection write failed during update: {e}")); + } + record_projection_update_success( + tenant, + entity_type, + operation, + source, + sequence_nr, + projection_started_at, + ); + } + + Ok(response) + } +} diff --git a/crates/temper-server/src/state/entity_ops/generation_access.rs b/crates/temper-server/src/state/entity_ops/generation_access.rs new file mode 100644 index 000000000..9e0f57e73 --- /dev/null +++ b/crates/temper-server/src/state/entity_ops/generation_access.rs @@ -0,0 +1,220 @@ +//! Generation-aware entity reads and creates. + +use super::*; + +impl ServerState { + /// Get the current state of an entity actor (legacy single-tenant). + #[deprecated(note = "Use `get_tenant_entity_state` with explicit tenant")] + pub async fn get_entity_state( + &self, + entity_type: &str, + entity_id: &str, + ) -> Result { + self.get_tenant_entity_state(&TenantId::default(), entity_type, entity_id) + .await + } + + /// Get the current state of an entity actor for a specific tenant. + #[instrument(skip_all, fields(otel.name = "entity.get_tenant_entity_state", tenant = %tenant, entity_type, entity_id))] + pub async fn get_tenant_entity_state( + &self, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + ) -> Result { + self.get_tenant_entity_state_with_context(tenant, entity_type, entity_id, None) + .await + } + + /// Get entity state inside a tenant generation already admitted by dispatch. + pub async fn get_tenant_entity_state_in_generation( + &self, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + agent_ctx: &AgentContext, + ) -> Result { + self.get_tenant_entity_state_with_context(tenant, entity_type, entity_id, Some(agent_ctx)) + .await + } + + pub(super) async fn get_tenant_entity_state_with_context( + &self, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + agent_ctx: Option<&AgentContext>, + ) -> Result { + let actor_ref = self + .get_or_spawn_tenant_actor_with_fields_in_context( + tenant, + entity_type, + entity_id, + serde_json::json!({}), + agent_ctx, + ) + .ok_or_else(|| { + format!("No transition table for tenant '{tenant}', entity type '{entity_type}'") + })?; + + // ADR-0048: retry transient ask failures (AskTimeout, MailboxFull) + // so a single slow actor reply does not surface as HTTP 500. + let policy = self.dispatch_retry_policy(); + retry::ask_with_backoff::<_, EntityResponse, _>(&actor_ref, || EntityMsg::GetState, &policy) + .await + .result + .map_err(|e| format!("Actor query failed: {e}")) + } + + /// Create a new entity with initial fields and return its state. + #[instrument(skip_all, fields(otel.name = "entity.get_or_create_tenant_entity", tenant = %tenant, entity_type, entity_id))] + pub async fn get_or_create_tenant_entity( + &self, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + initial_fields: serde_json::Value, + ) -> Result { + self.get_or_create_tenant_entity_with_context( + tenant, + entity_type, + entity_id, + initial_fields, + None, + ) + .await + } + + /// Create or resolve an entity inside an already-admitted tenant generation. + pub async fn get_or_create_tenant_entity_in_generation( + &self, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + initial_fields: serde_json::Value, + agent_ctx: &AgentContext, + ) -> Result { + self.get_or_create_tenant_entity_with_context( + tenant, + entity_type, + entity_id, + initial_fields, + Some(agent_ctx), + ) + .await + } + + pub(super) async fn get_or_create_tenant_entity_with_context( + &self, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + initial_fields: serde_json::Value, + agent_ctx: Option<&AgentContext>, + ) -> Result { + let actor_ref = self + .get_or_spawn_tenant_actor_with_fields_in_context( + tenant, + entity_type, + entity_id, + initial_fields, + agent_ctx, + ) + .ok_or_else(|| { + format!("No transition table for tenant '{tenant}', entity type '{entity_type}'") + })?; + + let policy = self.dispatch_retry_policy(); + let response = retry::ask_with_backoff::<_, EntityResponse, _>( + &actor_ref, + || EntityMsg::GetState, + &policy, + ) + .await + .result + .map_err(|e| format!("Actor query failed: {e}"))?; + + // Broadcast entity creation event for SSE subscribers + let seq = self.next_entity_event_sequence(tenant.as_str(), entity_type, entity_id); + let change = EntityStateChange { + seq, + entity_type: entity_type.to_string(), + entity_id: entity_id.to_string(), + action: "Created".to_string(), + status: response.state.status.clone(), + tenant: tenant.to_string(), + agent_id: None, + session_id: None, + intent: None, + observation_metadata: None, + }; + self.record_entity_observe_event_with_seq( + tenant.as_str(), + entity_type, + entity_id, + seq, + "state_change", + serde_json::to_value(&change).unwrap_or_default(), + ); + let _ = self.event_tx.send(change); + + if let Some(query_plane) = self.query_plane_store() { + let status = response.state.status.clone(); + let fields = self.query_projection_fields(tenant, entity_type, &response.state.fields); + let projected_state = self.query_projection_state(&response.state); + let sequence_nr = response.state.sequence_nr; + let operation = "upsert"; + let source = "create"; + record_projection_update_started(tenant, entity_type, operation, source); + let projection_started_at = Instant::now(); // determinism-ok: production-only projection duration metric + if let Err(e) = query_plane + .upsert_projection( + tenant.as_str(), + entity_type, + entity_id, + &status, + &fields, + &projected_state, + sequence_nr, + ) + .await + { + record_projection_update_error( + tenant, + entity_type, + operation, + source, + projection_started_at, + ); + // Surface the projection failure instead of swallowing it. + // Previously this was a `warn` followed by `Ok(response)`, + // which produced HTTP 201 even when the entity wasn't + // discoverable via $filter — silently corrupting any caller + // that does write-then-read-back. Returning Err propagates + // up through the OData write handler to a 5xx so the client + // knows to retry. The event is already durable in the + // events table; on retry the actor's idempotent UPSERT into + // entity_catalog/entity_field_index will succeed (or fail + // again loudly). + tracing::error!( + error = %e, + tenant = %tenant, + entity_type = %entity_type, + entity_id = %entity_id, + "failed to update query projection during create" + ); + return Err(format!("query projection write failed during create: {e}")); + } + record_projection_update_success( + tenant, + entity_type, + operation, + source, + sequence_nr, + projection_started_at, + ); + } + + Ok(response) + } +} diff --git a/crates/temper-server/src/state/entity_ops/key_activation.rs b/crates/temper-server/src/state/entity_ops/key_activation.rs new file mode 100644 index 000000000..0ac594d9e --- /dev/null +++ b/crates/temper-server/src/state/entity_ops/key_activation.rs @@ -0,0 +1,393 @@ +//! Declared-key activation and projection backfill entry points. + +use super::*; + +impl ServerState { + /// Populate the durable query-plane projections for collection reads. + /// + /// Two-phase approach: + /// 1. **Snapshot pass** — cheap: deserialises snapshots for entities that have them. + /// 2. **Persistence replay pass** — reconstructs state directly from the event log. + /// + /// Runs once as a background task after startup. New entities created after + /// boot are indexed via `run_post_dispatch_effects` step 8. + #[instrument(skip_all, fields(otel.name = "entity.populate_field_index", tenant = %tenant))] + pub async fn populate_field_index_from_snapshots(&self, tenant: &TenantId) { + projection_backfill::populate_field_index_from_snapshots(self, tenant).await; + } + + /// Backfill `entity_key_index` for declared-key entity types (ADR-0153), so a + /// keyed read can authoritatively prove absence for pre-existing entities. + /// + /// Independent of [`Self::populate_field_index_from_snapshots`]: the declared + /// key is `K` (1–3) tiny rows per entity, far cheaper than the broad `S`-wide + /// field-index re-scan, so it is gated and scheduled on its own rather than + /// riding the expensive projection backfill. Runs once as a background task; + /// entities written after boot are keyed inline at write time. + #[instrument(skip_all, fields(otel.name = "entity.populate_key_index", tenant = %tenant))] + pub async fn populate_key_index_from_snapshots(&self, tenant: &TenantId) { + projection_backfill::populate_key_index_from_snapshots(self, tenant).await; + } + + /// Fence declared-key contracts before publishing replacement specs. + /// + /// This invalidates an older coverage proof synchronously with activation + /// preparation. An empty contract also purges the type's old ownership rows + /// atomically, so a rapid A -> none -> A cycle cannot resurrect either the + /// original watermark or claims that should have been released. + pub async fn prepare_key_index_contracts_for_spec_activation( + &self, + publication_guard: &SpecPublicationGuard, + tenant: &TenantId, + ioa_sources: &[(&str, &str)], + ) -> Result { + self.prepare_key_index_contracts_for_spec_activation_with_removals( + publication_guard, + tenant, + ioa_sources, + &[], + ) + .await + } + + /// Fence a complete replacement, including entity types omitted by the new + /// registry. Removed types activate an empty contract and purge their + /// ownership rows in the same backend transaction as changed types. + pub async fn prepare_key_index_contracts_for_spec_activation_with_removals( + &self, + publication_guard: &SpecPublicationGuard, + tenant: &TenantId, + ioa_sources: &[(&str, &str)], + removed_entity_types: &[String], + ) -> Result { + publication_guard.validates(tenant)?; + let Some(storage) = self.storage_stack.as_ref() else { + return Ok(KeyContractActivationCutover::empty()); + }; + let store = &storage.events; + if !store.supports_authoritative_key_index() { + return Ok(KeyContractActivationCutover::empty()); + } + + let mut contracts = BTreeMap::new(); + let mut candidate_tables = Vec::new(); + for (entity_type, ioa_source) in ioa_sources { + if contracts.contains_key(*entity_type) { + return Err(format!( + "duplicate entity type {entity_type} in key contract activation" + )); + } + let table = temper_jit::table::TransitionTable::try_from_ioa_source(ioa_source) + .map_err(|error| { + format!( + "failed to prepare key contract for {entity_type} before spec activation: {error}" + ) + })?; + contracts.insert( + (*entity_type).to_string(), + ( + crate::key_index::declared_key_set_signature(&table.keys), + format!("{:x}", Sha256::digest(ioa_source.as_bytes())), + table.keys.is_empty(), + ), + ); + candidate_tables.push(((*entity_type).to_string(), table)); + } + for entity_type in removed_entity_types { + if contracts.contains_key(entity_type) { + return Err(format!( + "entity type {entity_type} cannot be both published and removed" + )); + } + let removed_fingerprint = format!( + "{:x}", + Sha256::digest(format!("temper.removed-spec.v1:{entity_type}").as_bytes()) + ); + contracts.insert( + entity_type.clone(), + ( + crate::key_index::declared_key_set_signature(&[]), + removed_fingerprint, + true, + ), + ); + } + + let activations = contracts + .into_iter() + .map( + |(entity_type, (key_set, spec_fingerprint, purge_existing_rows))| { + temper_runtime::persistence::KeyContractActivation { + entity_type, + key_set, + spec_fingerprint, + purge_existing_rows, + } + }, + ) + .collect::>(); + let affected_entity_types = activations + .iter() + .map(|activation| activation.entity_type.clone()) + .collect::>(); + { + let mut activating = self + .activating_key_contracts + .write() + .expect("activating key contracts lock poisoned"); + activating.extend( + affected_entity_types + .iter() + .map(|entity_type| (tenant.as_str().to_string(), entity_type.clone())), + ); + } + let activation_epochs = match store + .activate_key_index_contracts(tenant.as_str(), &activations) + .await + { + Ok(epochs) => epochs, + Err(error) => { + return Err(format!( + "failed to atomically activate key contracts for {tenant}: {error}" + )); + } + }; + for (entity_type, table) in &mut candidate_tables { + if let Some(epoch) = activation_epochs.get(entity_type) { + table.key_contract_activation_epoch = *epoch; + } + } + for entity_type in activation_epochs.keys() { + self.key_index_backfilled + .write() + .expect("key index backfilled lock poisoned") + .remove(&format!("{tenant}:{entity_type}")); + } + let prepared_coverage = projection_backfill::prepare_key_index_coverage_for_activation( + self, + tenant, + &candidate_tables, + ) + .await?; + Ok(KeyContractActivationCutover { + activation_epochs, + candidate_tables, + prepared_coverage, + affected_entity_types, + }) + } + + /// Complete a prepared cutover after the registry has published the supplied + /// epochs. Spawn remains gated until every candidate coverage CAS succeeds. + pub async fn finish_key_index_contract_activation( + &self, + publication_guard: &mut SpecPublicationGuard, + tenant: &TenantId, + cutover: &mut KeyContractActivationCutover, + ) -> Result<(), String> { + publication_guard.validates(tenant)?; + self.evict_key_contract_actors(tenant, &cutover.affected_entity_types); + const REPAIR_ATTEMPTS: usize = 3; + let mut last_error = None; + for attempt in 1..=REPAIR_ATTEMPTS { + match projection_backfill::publish_prepared_key_index_coverage( + self, + tenant, + &cutover.prepared_coverage, + ) + .await + { + Ok(()) => { + let mut activating = self + .activating_key_contracts + .write() + .expect("activating key contracts lock poisoned"); + for entity_type in &cutover.affected_entity_types { + activating.remove(&(tenant.as_str().to_string(), entity_type.clone())); + } + return Ok(()); + } + Err(error) if attempt < REPAIR_ATTEMPTS => { + last_error = Some(error); + cutover.prepared_coverage = + projection_backfill::prepare_key_index_coverage_for_activation( + self, + tenant, + &cutover.candidate_tables, + ) + .await?; + } + Err(error) => last_error = Some(error), + } + } + Err(format!( + "key contract activation for {tenant} remained not-ready after {REPAIR_ATTEMPTS} source-fenced repairs: {}", + last_error.unwrap_or_else(|| "unknown readiness failure".to_string()) + )) + } + + /// Publish every non-key runtime dependency, evict any actor that crossed + /// the arm/slow-spawn boundary, and reopen the tenant as the final cutover + /// step. Callers must not invoke this until registry, reactions, policies, + /// and required integration modules match the durable generation. + pub fn complete_spec_publication( + &self, + publication_guard: &mut SpecPublicationGuard, + tenant: &TenantId, + ) -> Result<(), String> { + self.complete_spec_publication_inner(publication_guard, tenant, false) + } + + /// Reopen a tenant after an exact retry of the complete runtime-generation + /// intent that originally left it fail-closed. Partial repair helpers must + /// use [`Self::complete_spec_publication`] so they cannot discharge debt for + /// missing policy, WASM, or unrelated durable specs. + pub fn complete_spec_publication_retry( + &self, + publication_guard: &mut SpecPublicationGuard, + tenant: &TenantId, + ) -> Result<(), String> { + self.complete_spec_publication_inner(publication_guard, tenant, true) + } + + pub(super) fn complete_spec_publication_inner( + &self, + publication_guard: &mut SpecPublicationGuard, + tenant: &TenantId, + allow_inherited_debt: bool, + ) -> Result<(), String> { + publication_guard.validates(tenant)?; + self.evict_tenant_actors(tenant); + publication_guard.release(allow_inherited_debt) + } + + pub(super) fn evict_key_contract_actors( + &self, + tenant: &TenantId, + affected_entity_types: &std::collections::BTreeSet, + ) { + self.evict_tenant_actors_for_types(tenant, Some(affected_entity_types)); + } + + pub(super) fn evict_tenant_actors(&self, tenant: &TenantId) { + self.evict_tenant_actors_for_types(tenant, None); + } + + pub(super) fn evict_tenant_actors_for_types( + &self, + tenant: &TenantId, + affected_entity_types: Option<&std::collections::BTreeSet>, + ) { + let removed = { + let mut actors = self + .actor_registry + .write() + .expect("actor registry lock poisoned"); + let keys = actors + .keys() + .filter(|actor_key| { + temper_runtime::tenant::parse_persistence_id_parts(actor_key) + .ok() + .is_some_and(|(actor_tenant, entity_type, _)| { + actor_tenant == tenant.as_str() + && affected_entity_types + .is_none_or(|types| types.contains(entity_type)) + }) + }) + .cloned() + .collect::>(); + keys.into_iter() + .filter_map(|key| actors.remove(&key).map(|actor| (key, actor))) + .collect::>() + }; + for (_, actor) in &removed { + let _ = actor.stop(); + } + if let Ok(mut last_accessed) = self.last_accessed.write() { + for (actor_key, _) in removed { + last_accessed.remove(&actor_key); + } + } + } + + /// Activate every currently registered key contract for a tenant before + /// startup begins serving traffic. Activated types omitted by the durable + /// registry are retired to an empty contract, closing both persist-first + /// crash boundaries without reviving older ownership authority. + pub async fn activate_registered_key_contracts(&self, tenant: &TenantId) -> Result<(), String> { + let mut publication_guard = self.begin_spec_publication(tenant).await?; + let sources = { + let registry = self.registry.read().expect("spec registry lock poisoned"); + registry + .entity_types(tenant) + .into_iter() + .filter_map(|entity_type| { + registry + .get_spec(tenant, entity_type) + .map(|spec| (entity_type.to_string(), spec.ioa_source.clone())) + }) + .collect::>() + }; + let intent_components = sources + .iter() + .map(|(entity_type, source)| (entity_type.as_str(), source.as_bytes())) + .collect::>(); + let intent = Self::spec_publication_intent("startup-full-registry", intent_components); + self.arm_spec_publication(&mut publication_guard, tenant, &intent)?; + let registered_types = sources + .iter() + .map(|(entity_type, _)| entity_type.as_str()) + .collect::>(); + let removed_entity_types = if let Some(storage) = self.storage_stack.as_ref() { + storage + .events + .key_index_activated_contracts() + .await + .map_err(|error| { + format!("failed to enumerate activated key contracts at startup: {error}") + })? + .into_iter() + .filter(|(activated_tenant, entity_type)| { + activated_tenant == tenant.as_str() + && !registered_types.contains(entity_type.as_str()) + }) + .map(|(_, entity_type)| entity_type) + .collect::>() + } else { + Vec::new() + }; + let source_refs = sources + .iter() + .map(|(entity_type, source)| (entity_type.as_str(), source.as_str())) + .collect::>(); + let mut cutover = self + .prepare_key_index_contracts_for_spec_activation_with_removals( + &publication_guard, + tenant, + &source_refs, + &removed_entity_types, + ) + .await?; + let live_tables = { + let registry = self.registry.read().expect("spec registry lock poisoned"); + cutover + .activation_epochs + .iter() + .filter_map(|(entity_type, epoch)| { + registry + .get_table_live(tenant, entity_type) + .map(|table| (table, *epoch)) + }) + .collect::>() + }; + for (table, epoch) in live_tables { + table + .write() + .expect("transition table lock poisoned") + .key_contract_activation_epoch = epoch; + } + self.finish_key_index_contract_activation(&mut publication_guard, tenant, &mut cutover) + .await?; + self.complete_spec_publication_retry(&mut publication_guard, tenant) + } +} diff --git a/crates/temper-server/src/state/entity_ops/key_index_coverage.rs b/crates/temper-server/src/state/entity_ops/key_index_coverage.rs new file mode 100644 index 000000000..721d6ff9b --- /dev/null +++ b/crates/temper-server/src/state/entity_ops/key_index_coverage.rs @@ -0,0 +1,184 @@ +//! Durable key-index coverage and revision fencing. + +use super::*; + +impl ServerState { + /// Hydrate the per-tenant `entity_key_index` watermark cache once from the durable + /// watermark (ADR-0153). Safe to call repeatedly; conservative on any failure (leaves + /// the type uncovered → a keyed miss falls back to the scan, never a wrong "absent"). + async fn ensure_key_index_watermarks_loaded(&self, tenant: &TenantId) { + let already_loaded = self + .key_index_watermarks_loaded + .read() + .expect("key index watermarks-loaded lock poisoned") + .contains(tenant.as_str()); + if already_loaded { + return; + } + if let Some((store, _)) = self.event_journal() { + if !store.supports_authoritative_key_index() { + self.key_index_watermarks_loaded + .write() + .expect("key index watermarks-loaded lock poisoned") + .insert(tenant.to_string()); + return; + } + if let Ok(types) = store.key_index_backfilled_types(tenant.as_str()).await { + let mut cache = self + .key_index_backfilled + .write() + .expect("key index backfilled lock poisoned"); + for (et, key_set) in types { + cache.insert(format!("{tenant}:{et}"), key_set); + } + } + // Mark loaded even if the query errored: an error means "no authority + // yet", which is the safe scan-fallback state, and a completing + // backfill sets the cache entry directly via `mark_key_index_backfilled`. + self.key_index_watermarks_loaded + .write() + .expect("key index watermarks-loaded lock poisoned") + .insert(tenant.to_string()); + } + } + + /// The declared key-set the `entity_key_index` backfill covered for `(tenant, + /// entity_type)`, or `None` if the type was never backfilled. The value is the + /// versioned declared-key signature (see [`declared_key_set_signature`]). + pub(crate) async fn key_index_backfill_covered_key_set( + &self, + tenant: &TenantId, + entity_type: &str, + ) -> Option { + if self.key_contract_activation_gated(tenant, entity_type) { + return None; + } + self.ensure_key_index_watermarks_loaded(tenant).await; + let cache_key = format!("{tenant}:{entity_type}"); + let (store, _) = self.event_journal()?; + if !store.supports_authoritative_key_index() { + return None; + } + + // Live writes invalidate a stale durable watermark inside their journal + // transaction. Refresh this exact decision before every keyed ownership + // proof so an in-process A -> B -> A spec cycle cannot resurrect a cached A + // watermark after the B write removed it. + let covered = match store.key_index_backfilled_types(tenant.as_str()).await { + Ok(types) => types + .into_iter() + .find_map(|(found_type, key_set)| (found_type == entity_type).then_some(key_set)), + Err(error) => { + tracing::warn!( + tenant = %tenant, entity_type, error = %error, + "failed to refresh key-index coverage; treating type as scan-only" + ); + None + } + }; + let mut cache = self + .key_index_backfilled + .write() + .expect("key index backfilled lock poisoned"); + if let Some(key_set) = covered.as_ref() { + cache.insert(cache_key, key_set.clone()); + } else { + cache.remove(&cache_key); + } + covered + } + + /// Whether `entity_key_index` is complete for `(tenant, entity_type)` under the + /// CURRENT declared key-set — the ADR-0153/0171 backfill watermark. True only + /// when the covered signature equals `current_key_set`, so a keyed hit or miss is + /// trusted only after every currently-declared key is reconciled. Conservative on + /// any failure (returns false → authoritative scan, never stale ownership). + pub(crate) async fn key_index_backfill_complete( + &self, + tenant: &TenantId, + entity_type: &str, + current_key_set: &str, + ) -> bool { + let Some((store, _)) = self.event_journal() else { + return false; + }; + if !store.supports_authoritative_key_index() { + return false; + } + self.key_index_backfill_covered_key_set(tenant, entity_type) + .await + .as_deref() + == Some(current_key_set) + } + + /// Record (durably + in the read-path cache) that `entity_key_index` is complete + /// for `(tenant, entity_type)` covering exactly `key_set` (the versioned declared- + /// key signature). Called by the backfill once it has keyed every existing + /// entity of the type, so subsequent keyed hits and misses are bounded without + /// scanning (ADR-0153/0171). Overwrites any stale earlier signature. + #[cfg(test)] + pub(crate) async fn mark_key_index_backfilled( + &self, + tenant: &TenantId, + entity_type: &str, + key_set: &str, + ) { + let Some((store, _)) = self.event_journal() else { + return; + }; + if !store.supports_authoritative_key_index() { + return; + } + if let Err(e) = store + .mark_key_index_backfilled(tenant.as_str(), entity_type, key_set) + .await + { + tracing::error!( + tenant = %tenant, entity_type, error = %e, + "failed to persist key-index backfill watermark" + ); + return; + } + self.key_index_backfilled + .write() + .expect("key index backfilled lock poisoned") + .insert(format!("{tenant}:{entity_type}"), key_set.to_string()); + } + + /// Publish a backfill watermark only if no live write changed the type's key + /// contract while replay was in progress. The cache changes only after the + /// durable compare-and-set succeeds. + pub(crate) async fn mark_key_index_backfilled_if_revision( + &self, + tenant: &TenantId, + entity_type: &str, + key_set: &str, + expected_revision: u64, + ) -> Result { + let Some((store, _)) = self.event_journal() else { + return Ok(false); + }; + if !store.supports_authoritative_key_index() { + return Ok(false); + } + let published = store + .mark_key_index_backfilled_if_revision( + tenant.as_str(), + entity_type, + key_set, + expected_revision, + ) + .await?; + let cache_key = format!("{tenant}:{entity_type}"); + let mut cache = self + .key_index_backfilled + .write() + .expect("key index backfilled lock poisoned"); + if published { + cache.insert(cache_key, key_set.to_string()); + } else { + cache.remove(&cache_key); + } + Ok(published) + } +} diff --git a/crates/temper-server/src/state/entity_ops/status_resolution.rs b/crates/temper-server/src/state/entity_ops/status_resolution.rs new file mode 100644 index 000000000..83ed311b5 --- /dev/null +++ b/crates/temper-server/src/state/entity_ops/status_resolution.rs @@ -0,0 +1,60 @@ +//! Generation-aware entity-status resolution. + +use super::*; + +/// Resolve the current status of an entity. +/// +/// Fast path: check the `entity_state_cache` (populated on every successful dispatch). +/// Slow path: fall back to `get_tenant_entity_state()` (async actor ask) and backfill cache. +impl ServerState { + #[instrument(skip_all, fields(otel.name = "entity.resolve_entity_status", tenant = %tenant, entity_type, entity_id))] + pub async fn resolve_entity_status( + &self, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + ) -> Option { + self.resolve_entity_status_with_context(tenant, entity_type, entity_id, None) + .await + } + + /// Resolve entity status inside an already-admitted tenant generation. + pub(crate) async fn resolve_entity_status_in_generation( + &self, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + agent_ctx: &AgentContext, + ) -> Option { + self.resolve_entity_status_with_context(tenant, entity_type, entity_id, Some(agent_ctx)) + .await + } + + pub(super) async fn resolve_entity_status_with_context( + &self, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + agent_ctx: Option<&AgentContext>, + ) -> Option { + // Fast path: check cache (LruCache::get requires &mut, so use Mutex). + let cache_key = format!("{tenant}:{entity_type}:{entity_id}"); + if let Ok(mut cache) = self.entity_state_cache.lock() + && let Some((status, _timestamp)) = cache.get(&cache_key) + { + return Some(status.clone()); + } + + // Slow path: actor ask + backfill + if let Ok(response) = self + .get_tenant_entity_state_with_context(tenant, entity_type, entity_id, agent_ctx) + .await + { + let status = response.state.status.clone(); + self.cache_entity_status(cache_key, status.clone()); + Some(status) + } else { + None + } + } +} diff --git a/crates/temper-server/src/state/file_initial_writes.rs b/crates/temper-server/src/state/file_initial_writes.rs index 90ba4df9c..015945dbe 100644 --- a/crates/temper-server/src/state/file_initial_writes.rs +++ b/crates/temper-server/src/state/file_initial_writes.rs @@ -1,14 +1,20 @@ use std::sync::{Arc, RwLock}; -use temper_runtime::persistence::{EventMetadata, PersistenceEnvelope, PersistenceError}; -use temper_runtime::scheduler::{sim_now, sim_uuid}; +use temper_runtime::persistence::{IndexReconciliation, PersistenceError}; +use temper_runtime::scheduler::sim_now; -use crate::entity_actor::{EntityEvent, EntityResponse, EntityState, process_action_with_xref}; +use crate::entity_actor::{EntityEvent, EntityResponse}; use crate::events::EntityStateChange; use super::file_writes::content_hash_and_native_blob_key; use super::{FileStreamContentError, ServerState}; +mod synthetic; + +use synthetic::{ + apply_synthetic_file_action, initial_file_state, push_synthetic_event, synthetic_envelope, +}; + impl ServerState { /// Create a brand-new TemperFS `File` and persist its first stream bytes as /// one kernel operation. @@ -52,7 +58,10 @@ impl ServerState { mime_type: &str, agent_ctx: &crate::request_context::AgentContext, ) -> Result { - if self.ensure_entity_loaded(tenant, "File", file_id).await { + if self + .ensure_entity_loaded_in_generation(tenant, "File", file_id, agent_ctx) + .await + { return Err(FileStreamContentError::ActionRejected(format!( "File('{file_id}') already exists; use File $value update for existing content" ))); @@ -73,7 +82,7 @@ impl ServerState { .and_then(|value| value.as_str()) .unwrap_or_default() .to_string(); - self.reject_write_if_workspace_not_active(tenant, &workspace_id) + self.reject_write_if_workspace_not_active(tenant, &workspace_id, agent_ctx) .await?; let (content_hash, blob_key) = content_hash_and_native_blob_key(body); @@ -148,9 +157,36 @@ impl ServerState { .map(|(idx, event)| synthetic_envelope(&persistence_id, (idx + 1) as u64, event)) .collect::, _>>()?; - match store.append(&persistence_id, 0, &envelopes).await { + // Exact reconciliation participates even for a no-key contract so a + // hot spec change cannot strand ownership rows from the prior version. + let reconcile_keys = true; + let key_rows = crate::key_index::derive_entity_key_rows( + &table.keys, + &state.fields, + state.status != "Deleted", + ); + let key_set_signature = crate::key_index::declared_key_write_contract(&table); + match store + .append_with_index_rows( + &persistence_id, + 0, + &envelopes, + &key_rows, + &[], + IndexReconciliation { + keys: reconcile_keys, + key_set_signature: Some(key_set_signature), + vectors: false, + snapshot_source: temper_runtime::persistence::SnapshotSourceFence::Absent, + }, + ) + .await + { Ok(sequence_nr) => state.sequence_nr = sequence_nr, - Err(PersistenceError::ConcurrencyViolation { .. }) => { + Err( + PersistenceError::ConcurrencyViolation { .. } + | PersistenceError::SnapshotGenerationChanged, + ) => { return Err(FileStreamContentError::ActionRejected(format!( "File('{file_id}') already exists; use File $value update for existing content" ))); @@ -317,10 +353,27 @@ impl ServerState { let file_id = file_id.to_string(); let to_state = response.state.status.clone(); let fields = response.state.fields.clone(); - let agent_ctx = agent_ctx.clone(); + let agent_ctx = agent_ctx + .fork_tenant_generation_lease(&tenant) + .unwrap_or_else(|| { + let mut detached = agent_ctx.clone(); + detached.detach_tenant_generation_lease(); + detached + }); tokio::spawn(async move { // determinism-ok: post-commit reaction side effects mirror the // existing non-awaited File `$value` update behavior. + let Some(agent_ctx) = state + .activate_immediate_tenant_work(&tenant, agent_ctx) + .await + else { + tracing::error!( + tenant = %tenant, + entity_id = %file_id, + "File initial-write reactions could not enter a complete runtime generation" + ); + return; + }; dispatcher .dispatch_reactions( &state, @@ -338,92 +391,3 @@ impl ServerState { } } } - -fn initial_file_state( - file_id: &str, - table: &temper_jit::table::TransitionTable, - initial_fields: serde_json::Value, -) -> EntityState { - let mut fields = initial_fields; - if let Some(obj) = fields.as_object_mut() { - obj.entry("Id".to_string()) - .or_insert(serde_json::Value::String(file_id.to_string())); - obj.entry("Status".to_string()) - .or_insert(serde_json::Value::String(table.initial_state.clone())); - } - - EntityState { - entity_type: "File".to_string(), - entity_id: file_id.to_string(), - status: table.initial_state.clone(), - item_count: 0, - counters: std::collections::BTreeMap::new(), - booleans: std::collections::BTreeMap::new(), - lists: std::collections::BTreeMap::new(), - fields, - events: std::collections::VecDeque::new(), - total_event_count: 0, - events_since_snapshot: 0, - last_snapshot_sequence_nr: 0, - sequence_nr: 0, - processed_idempotency_keys: std::collections::BTreeMap::new(), - } -} - -fn apply_synthetic_file_action( - state: &mut EntityState, - table: &temper_jit::table::TransitionTable, - action: &str, - params: serde_json::Value, - cross_entity_booleans: &std::collections::BTreeMap, -) -> Result { - let result = process_action_with_xref(state, table, action, ¶ms, cross_entity_booleans); - if !result.overflow_blobs.is_empty() { - return Err(FileStreamContentError::State(format!( - "File.{action} produced field-overflow blobs on the atomic initial content path" - ))); - } - if !result.success { - return Err(FileStreamContentError::ActionRejected( - result - .error - .unwrap_or_else(|| format!("File.{action} rejected")), - )); - } - result.event.ok_or_else(|| { - FileStreamContentError::State(format!( - "File.{action} succeeded without producing a durable event" - )) - }) -} - -fn push_synthetic_event( - state: &mut EntityState, - events: &mut Vec, - event: EntityEvent, -) { - state.sequence_nr = state.sequence_nr.saturating_add(1); - state.push_event_bounded(event.clone()); - events.push(event); -} - -fn synthetic_envelope( - persistence_id: &str, - sequence_nr: u64, - event: &EntityEvent, -) -> Result { - let payload = serde_json::to_value(event) - .map_err(|e| FileStreamContentError::State(format!("failed to serialize event: {e}")))?; - Ok(PersistenceEnvelope { - sequence_nr, - event_type: event.action.clone(), - payload, - metadata: EventMetadata { - event_id: sim_uuid(), - causation_id: sim_uuid(), - correlation_id: sim_uuid(), - timestamp: event.timestamp, - actor_id: persistence_id.to_string(), - }, - }) -} diff --git a/crates/temper-server/src/state/file_initial_writes/synthetic.rs b/crates/temper-server/src/state/file_initial_writes/synthetic.rs new file mode 100644 index 000000000..07aee886c --- /dev/null +++ b/crates/temper-server/src/state/file_initial_writes/synthetic.rs @@ -0,0 +1,95 @@ +use temper_runtime::persistence::{EventMetadata, PersistenceEnvelope}; +use temper_runtime::scheduler::sim_uuid; + +use crate::entity_actor::{EntityEvent, EntityState, process_action_with_xref}; + +use super::super::FileStreamContentError; + +pub(super) fn initial_file_state( + file_id: &str, + table: &temper_jit::table::TransitionTable, + initial_fields: serde_json::Value, +) -> EntityState { + let mut fields = initial_fields; + if let Some(obj) = fields.as_object_mut() { + obj.entry("Id".to_string()) + .or_insert(serde_json::Value::String(file_id.to_string())); + obj.entry("Status".to_string()) + .or_insert(serde_json::Value::String(table.initial_state.clone())); + } + + EntityState { + entity_type: "File".to_string(), + entity_id: file_id.to_string(), + status: table.initial_state.clone(), + item_count: 0, + counters: std::collections::BTreeMap::new(), + booleans: std::collections::BTreeMap::new(), + lists: std::collections::BTreeMap::new(), + fields, + events: std::collections::VecDeque::new(), + total_event_count: 0, + events_since_snapshot: 0, + last_snapshot_sequence_nr: 0, + sequence_nr: 0, + processed_idempotency_keys: std::collections::BTreeMap::new(), + } +} + +pub(super) fn apply_synthetic_file_action( + state: &mut EntityState, + table: &temper_jit::table::TransitionTable, + action: &str, + params: serde_json::Value, + cross_entity_booleans: &std::collections::BTreeMap, +) -> Result { + let result = process_action_with_xref(state, table, action, ¶ms, cross_entity_booleans); + if !result.overflow_blobs.is_empty() { + return Err(FileStreamContentError::State(format!( + "File.{action} produced field-overflow blobs on the atomic initial content path" + ))); + } + if !result.success { + return Err(FileStreamContentError::ActionRejected( + result + .error + .unwrap_or_else(|| format!("File.{action} rejected")), + )); + } + result.event.ok_or_else(|| { + FileStreamContentError::State(format!( + "File.{action} succeeded without producing a durable event" + )) + }) +} + +pub(super) fn push_synthetic_event( + state: &mut EntityState, + events: &mut Vec, + event: EntityEvent, +) { + state.sequence_nr = state.sequence_nr.saturating_add(1); + state.push_event_bounded(event.clone()); + events.push(event); +} + +pub(super) fn synthetic_envelope( + persistence_id: &str, + sequence_nr: u64, + event: &EntityEvent, +) -> Result { + let payload = serde_json::to_value(event) + .map_err(|e| FileStreamContentError::State(format!("failed to serialize event: {e}")))?; + Ok(PersistenceEnvelope { + sequence_nr, + event_type: event.action.clone(), + payload, + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: event.timestamp, + actor_id: persistence_id.to_string(), + }, + }) +} diff --git a/crates/temper-server/src/state/file_writes.rs b/crates/temper-server/src/state/file_writes.rs index 0e9cfe873..3def04e19 100644 --- a/crates/temper-server/src/state/file_writes.rs +++ b/crates/temper-server/src/state/file_writes.rs @@ -52,12 +52,13 @@ impl ServerState { &self, tenant: &temper_runtime::tenant::TenantId, workspace_id: &str, + agent_ctx: &crate::request_context::AgentContext, ) -> Result<(), FileStreamContentError> { if workspace_id.is_empty() { return Ok(()); } match self - .resolve_entity_status(tenant, "Workspace", workspace_id) + .resolve_entity_status_in_generation(tenant, "Workspace", workspace_id, agent_ctx) .await { Some(status) if status != "Active" => { @@ -155,7 +156,7 @@ impl ServerState { // happen if the workspace refuses the write (otherwise an orphaned blob // would be left behind on guard rejection). let file_state = self - .get_tenant_entity_state(tenant, "File", file_id) + .get_tenant_entity_state_in_generation(tenant, "File", file_id, agent_ctx) .await .map_err(|e| { FileStreamContentError::State(format!( @@ -169,7 +170,7 @@ impl ServerState { .get("workspace_id") .and_then(|value| value.as_str()) .unwrap_or_default(); - self.reject_write_if_workspace_not_active(tenant, workspace_id) + self.reject_write_if_workspace_not_active(tenant, workspace_id, agent_ctx) .await?; self.put_content_addressed_blob(tenant, &blob_key, body, None) @@ -233,7 +234,7 @@ impl ServerState { ) -> Result { let mut entity_state = serde_json::to_value( &self - .get_tenant_entity_state(tenant, "File", file_id) + .get_tenant_entity_state_in_generation(tenant, "File", file_id, agent_ctx) .await .map_err(|e| { FileStreamContentError::State(format!( diff --git a/crates/temper-server/src/state/mod.rs b/crates/temper-server/src/state/mod.rs index e9cd9796d..aeb2c485f 100644 --- a/crates/temper-server/src/state/mod.rs +++ b/crates/temper-server/src/state/mod.rs @@ -21,7 +21,9 @@ mod published_artifacts; mod query_projection_queue; pub(crate) mod rate_limit; mod runtime_metrics; +pub(crate) mod source_fenced_projection; pub(crate) mod storage_caps; +mod tenant_generation; pub mod trajectory; pub mod wasm_invocation_log; @@ -39,9 +41,11 @@ pub use persistence::WasmModuleSource; pub use policy_suggestions::PolicySuggestionEngine; pub use published_artifacts::PublishFileArtifactRequest; pub(crate) use query_projection_queue::{ProjectionEnqueueOutcome, QueryProjectionWriteQueue}; +pub use tenant_generation::{SpecPublicationGuard, TenantGenerationLease}; pub use trajectory::{TrajectoryEntry, TrajectorySource}; pub use wasm_invocation_log::WasmInvocationEntry; +use sha2::{Digest, Sha256}; use std::collections::{BTreeMap, BTreeSet}; use std::num::NonZeroUsize; use std::sync::{Arc, Mutex, OnceLock, RwLock}; @@ -87,10 +91,27 @@ pub struct BoundActionHookContext<'a> { pub action: &'a str, pub params: &'a serde_json::Value, pub state_json: &'a serde_json::Value, + /// Stable, content-bound operation ID. Implementations must treat repeated + /// calls with this ID as the same operation and return the same successful + /// output after any partial completion. + pub operation_id: &'a str, + /// Runtime-generation token captured while the governed request still held + /// its tenant read lease. Publication-capable hooks must use this token for + /// their first writer acquisition after the lease handoff. + pub expected_generation: Option, } #[async_trait::async_trait] pub trait BoundActionHook: Send + Sync { + /// Whether this hook must run after the request releases its stable tenant + /// generation because the hook itself may publish a replacement generation. + fn requires_generation_handoff(&self, _entity_type: &str, _action: &str) -> bool { + false + } + + /// Execute one content-idempotent operation. A process may repeat this call + /// after losing the completion acknowledgement, so external mutations and + /// the returned output must be stable for `ctx.operation_id`. async fn after_bound_action( &self, ctx: BoundActionHookContext<'_>, @@ -398,13 +419,28 @@ pub struct ServerState { pub entity_index_hydrated: Arc>>, /// `{tenant}:{entity_type}` keys whose `entity_key_index` backfill is complete /// `"tenant:entity_type" -> covered key-set` (ADR-0153 watermark cache). The value - /// is the sorted comma-joined declared key names the backfill covered. A keyed read - /// MISS on that type is authoritative absence ONLY when the covered key-set still - /// equals the currently-declared one — so a newly-declared, not-yet-backfilled key - /// never reads a present entity as absent (it falls back to the scan). This turns a - /// 413-scan into an O(log n) answer once the type is fully keyed (ARN-68). Loaded - /// lazily from the durable watermark (see [`key_index_watermarks_loaded`]). + /// is the versioned declared-key signature the backfill covered. A keyed hit or + /// miss is authoritative ONLY when the covered signature equals the current one; + /// otherwise the read uses authoritative replay. This turns a 413-scan into an + /// O(log n) answer once the type is fully keyed (ARN-68). Loaded lazily from the + /// durable watermark (see [`key_index_watermarks_loaded`]). pub key_index_backfilled: Arc>>, + /// Tenant/type contracts currently inside the fenced activate → replay → + /// publish → ready cutover. Actor spawn is closed until readiness commits. + pub(crate) activating_key_contracts: Arc>>, + /// Per-tenant coordinators serializing durable spec publication through + /// registry/key-contract readiness cutover. + spec_publication_locks: Arc>>>>, + /// Monotonic version of each complete live tenant generation. A bound + /// action captures this while holding a read lease and validates it after + /// handing off to a publication writer. + tenant_generation_versions: Arc>>, + /// Tenants whose publication may have crossed the durable commit boundary + /// but has not completed its live registry/key-contract cutover. + spec_publication_gated_tenants: Arc>>, + /// Exact runtime-generation intent responsible for a sticky tenant gate. + /// Only an identical retry may discharge an outcome-ambiguous publication. + spec_publication_debts: Arc>>, /// Tenants whose durable watermarks have been read into `key_index_backfilled` /// at least once this run. Gates the one-time-per-tenant load on the read path. pub key_index_watermarks_loaded: Arc>>, @@ -442,7 +478,10 @@ pub struct ServerState { /// LRU cache of entity current state, updated on every state change broadcast. /// Key: "{tenant}:{entity_type}:{entity_id}", Value: (current_state, last_updated). /// Capped at [`state_cache_budget()`] entries; oldest entry evicted automatically. - #[allow(clippy::type_complexity)] + #[expect( + clippy::type_complexity, + reason = "claim-scoped weak locks require the ownership shape to remain explicit" + )] pub entity_state_cache: Arc)>>>, /// Configurable timeout for actor ask operations (default: 5s). @@ -459,6 +498,12 @@ pub struct ServerState { pub eventual_tracker: Arc>, /// Idempotency cache for deduplicating agent retries. pub idempotency_cache: Arc, + /// Per-claim serializers for concurrent composite callbacks in this + /// single-process runtime. Values are weak so completed claim namespaces + /// are reclaimed on the next admission instead of becoming durable memory. + #[allow(clippy::type_complexity)] + pub(crate) composite_claim_locks: + Arc>>>>, /// Optional encrypted secrets vault for per-tenant secret management. /// Broadcast channel for new pending decisions (SSE subscriptions). pub pending_decision_tx: Arc>, @@ -683,6 +728,11 @@ impl ServerState { entity_index: Arc::new(RwLock::new(BTreeMap::new())), entity_index_hydrated: Arc::new(RwLock::new(BTreeSet::new())), key_index_backfilled: Arc::new(RwLock::new(BTreeMap::new())), + activating_key_contracts: Arc::new(RwLock::new(BTreeSet::new())), + spec_publication_locks: Arc::new(tokio::sync::Mutex::new(BTreeMap::new())), + tenant_generation_versions: Arc::new(RwLock::new(BTreeMap::new())), + spec_publication_gated_tenants: Arc::new(RwLock::new(BTreeSet::new())), + spec_publication_debts: Arc::new(RwLock::new(BTreeMap::new())), key_index_watermarks_loaded: Arc::new(RwLock::new(BTreeSet::new())), event_tx: Arc::new(event_tx), entity_observe_tx: Arc::new(entity_observe_tx), @@ -708,6 +758,7 @@ impl ServerState { crate::eventual_invariants::EventualInvariantTracker::new(), )), idempotency_cache: Arc::new(IdempotencyCache::new()), + composite_claim_locks: Arc::new(Mutex::new(BTreeMap::new())), pending_decision_tx: Arc::new(pending_decision_tx), tenant_policies: Arc::new(RwLock::new(BTreeMap::new())), commons_guardrail_tenants: Arc::new(RwLock::new(BTreeSet::new())), @@ -931,6 +982,11 @@ impl ServerState { entity_index: Arc::new(RwLock::new(BTreeMap::new())), entity_index_hydrated: Arc::new(RwLock::new(BTreeSet::new())), key_index_backfilled: Arc::new(RwLock::new(BTreeMap::new())), + activating_key_contracts: Arc::new(RwLock::new(BTreeSet::new())), + spec_publication_locks: Arc::new(tokio::sync::Mutex::new(BTreeMap::new())), + tenant_generation_versions: Arc::new(RwLock::new(BTreeMap::new())), + spec_publication_gated_tenants: Arc::new(RwLock::new(BTreeSet::new())), + spec_publication_debts: Arc::new(RwLock::new(BTreeMap::new())), key_index_watermarks_loaded: Arc::new(RwLock::new(BTreeSet::new())), event_tx: Arc::new(event_tx), entity_observe_tx: Arc::new(entity_observe_tx), @@ -956,6 +1012,7 @@ impl ServerState { crate::eventual_invariants::EventualInvariantTracker::new(), )), idempotency_cache: Arc::new(IdempotencyCache::new()), + composite_claim_locks: Arc::new(Mutex::new(BTreeMap::new())), pending_decision_tx: Arc::new(pending_decision_tx), tenant_policies: Arc::new(RwLock::new(BTreeMap::new())), commons_guardrail_tenants: Arc::new(RwLock::new(BTreeSet::new())), @@ -1209,7 +1266,7 @@ impl ServerState { let entity_state = serde_json::to_value( &self - .get_tenant_entity_state(tenant, "File", file_id) + .get_tenant_entity_state_in_generation(tenant, "File", file_id, agent_ctx) .await .map_err(|e| format!("failed to load File('{file_id}') state: {e}"))? .state, diff --git a/crates/temper-server/src/state/projection_backfill.rs b/crates/temper-server/src/state/projection_backfill.rs index 72f511c83..ed9ee660e 100644 --- a/crates/temper-server/src/state/projection_backfill.rs +++ b/crates/temper-server/src/state/projection_backfill.rs @@ -3,19 +3,32 @@ use std::time::Instant; use temper_runtime::tenant::TenantId; -use crate::entity_actor::recover_entity_state_from_store; -use crate::runtime_metrics; +use crate::entity_actor::{EntityRecoveryContext, recover_entity_state_from_stable_sources}; use super::ServerState; +mod entity_load; mod key_index; mod replay_parity; mod vector_index; -pub(super) use key_index::populate_key_index_from_snapshots; +use super::source_fenced_projection::repair_projection_from_stable_source; +use entity_load::{EntityLoadOutcome, load_entity_current_fields}; +pub(super) use key_index::{ + PreparedKeyIndexCoverage, populate_key_index_from_snapshots, + prepare_key_index_coverage_for_activation, publish_prepared_key_index_coverage, +}; pub(super) use replay_parity::verify_query_projection_replay_parity; pub(super) use vector_index::populate_vector_index_from_snapshots; +const MAX_FIELD_BACKFILL_SOURCE_ATTEMPTS: usize = 3; + +enum FieldBackfillOutcome { + Upserted { event_count: u64, sequence_nr: u64 }, + Removed { event_count: u64, sequence_nr: u64 }, + Empty { event_count: u64 }, +} + pub(super) fn transition_table_for( state: &ServerState, tenant: &TenantId, @@ -35,59 +48,6 @@ pub(super) fn transition_table_for( }) } -/// Outcome of loading one entity's current state for an index backfill (ADR-0153, -/// ADR-0155). Shared by the key and vector backfills so they classify entities the -/// same way — the distinction is the watermark soundness gate. -pub(super) enum EntityLoadOutcome { - /// Loaded — index it from these fields. - Fields(serde_json::Value), - /// Definitively skippable: deleted, or a phantom with no events. Correctly NOT - /// indexed, and NOT a failure (it must not block the watermark). - Skip, - /// The entity exists (it was enumerated from the durable store) but its current - /// state could not be loaded — no transition table to replay with, an unreadable - /// snapshot, or a replay error. Indexing it is impossible, so the type must NOT be - /// watermarked; otherwise a read would treat a present-but-unindexed entity as - /// authoritatively covered. This is the soundness gate. - LoadFailed, -} - -/// Load one entity's CURRENT state for an index backfill: snapshot if present and -/// readable, else strict event replay (so a field mutated after the last snapshot is -/// indexed at its current value, and a journal read failure fails the watermark -/// rather than silently "starting fresh"). -pub(super) async fn load_entity_current_fields( - tenant: &TenantId, - entity_type: &str, - entity_id: &str, - table: Option<&temper_jit::TransitionTable>, - store: &crate::storage::BoxedEventStore, - backend: crate::storage::BackendLabel, - blob_store: Option<&crate::blob_store::BlobStore>, -) -> EntityLoadOutcome { - let Some(table) = table else { - return EntityLoadOutcome::LoadFailed; - }; - match recover_entity_state_from_store( - tenant.as_str(), - entity_type, - entity_id, - table, - store, - backend, - &serde_json::json!({}), - blob_store, - true, // strict: a journal read failure → Err → LoadFailed (don't watermark) - ) - .await - { - Err(_) => EntityLoadOutcome::LoadFailed, - Ok(state) if state.status == "Deleted" => EntityLoadOutcome::Skip, - Ok(state) if state.total_event_count == 0 => EntityLoadOutcome::Skip, - Ok(state) => EntityLoadOutcome::Fields(state.fields), - } -} - /// Backfill the broad `entity_field_index` (every field of every entity) so the native /// AND-equality candidate pushdown can bound any non-keyed point lookup (e.g. `Path eq /// '/souls' and WorkspaceId eq …`) instead of full-scanning and 413ing at tenant scale @@ -102,9 +62,9 @@ pub(super) async fn populate_field_index_from_snapshots(state: &ServerState, ten let Some((store, backend)) = state.event_journal() else { return; }; - let Some(query_plane) = state.query_plane_store() else { + if state.query_plane_store().is_none() { return; - }; + } // Enumerate authoritatively: every entity type from the registry, and its entity // ids from `store.list_entity_ids_by_type`. It must NOT read `state.entity_index`, @@ -162,110 +122,14 @@ pub(super) async fn populate_field_index_from_snapshots(state: &ServerState, ten let mut indexed = 0usize; let mut errors = 0usize; - let mut needs_replay = Vec::new(); - - for (entity_type, entity_id) in &entities { - let persistence_id = format!("{tenant}:{entity_type}:{entity_id}"); - match store.load_snapshot(&persistence_id).await { - Ok(Some((_seq, snapshot_bytes))) => { - if let Ok(state_snapshot) = - serde_json::from_slice::(&snapshot_bytes) - { - if let Err(e) = query_plane - .upsert_projection( - tenant.as_str(), - entity_type, - entity_id, - &state_snapshot.status, - &state.query_projection_fields( - tenant, - entity_type, - &state_snapshot.fields, - ), - &state.query_projection_state(&state_snapshot), - state_snapshot.sequence_nr, - ) - .await - { - tracing::debug!( - error = %e, - entity_type = %entity_type, - entity_id = %entity_id, - "field index backfill: upsert failed" - ); - errors += 1; - crate::query_projection_metrics::record_backfill_entities( - tenant.as_str(), - entity_type, - "backfill_snapshot", - "error", - 1, - ); - } else { - indexed += 1; - crate::query_projection_metrics::record_backfill_entities( - tenant.as_str(), - entity_type, - "backfill_snapshot", - "ok", - 1, - ); - crate::query_projection_metrics::record_update_applied_sequence( - tenant.as_str(), - entity_type, - "upsert", - "backfill_snapshot", - state_snapshot.sequence_nr, - ); - } - } - } - Ok(None) => { - needs_replay.push((entity_type.clone(), entity_id.clone())); - crate::query_projection_metrics::record_backfill_entities( - tenant.as_str(), - entity_type, - "backfill_snapshot", - "missing_snapshot", - 1, - ); - } - Err(e) => { - tracing::debug!( - error = %e, - entity_type = %entity_type, - entity_id = %entity_id, - "field index backfill: snapshot load failed" - ); - errors += 1; - crate::query_projection_metrics::record_backfill_entities( - tenant.as_str(), - entity_type, - "backfill_snapshot", - "error", - 1, - ); - } - } - // Phase 1 now iterates EVERY entity (not the near-empty lazy index), so yield - // between entities for cooperative back-pressure on large tenants — mirroring - // phase 2 and the key-index backfill. - tokio::task::yield_now().await; - } + let needs_replay = entities; tracing::info!( tenant = %tenant, total, - snapshot_indexed = indexed, needs_replay = needs_replay.len(), - "field index phase 1 (snapshots) complete, starting phase 2 (persistence replay)" + "field index backfill starting stable snapshot/journal recovery" ); - if !needs_replay.is_empty() { - runtime_metrics::record_projection_backfill_snapshot_misses( - tenant.as_str(), - needs_replay.len() as u64, - ); - } for (entity_type, entity_id) in &needs_replay { let Some(table) = transition_table_for(state, tenant, entity_type) else { @@ -286,83 +150,127 @@ pub(super) async fn populate_field_index_from_snapshots(state: &ServerState, ten }; let tenant_blob_store = state.blob_store_for_tenant(tenant).ok(); - let replayed = recover_entity_state_from_store( - tenant.as_str(), - entity_type, - entity_id, - &table, - &store, - backend, - &serde_json::json!({}), - tenant_blob_store.as_ref(), - false, // field-index backfill: lenient (unchanged behavior) - ) - .await; - let replayed = match replayed { - Ok(state) => state, - Err(e) => { - tracing::debug!( - error = %e, - entity_type = %entity_type, - entity_id = %entity_id, - "field index backfill: persistence replay failed" - ); - errors += 1; - continue; + let persistence_id = format!("{tenant}:{entity_type}:{entity_id}"); + let mut outcome = None; + let mut last_event_count = 0; + for attempt in 1..=MAX_FIELD_BACKFILL_SOURCE_ATTEMPTS { + let source = match recover_entity_state_from_stable_sources(EntityRecoveryContext { + tenant: tenant.as_str(), + entity_type, + entity_id, + table: &table, + store: &store, + backend, + initial_fields: &serde_json::json!({}), + blob_store: tenant_blob_store.as_ref(), + }) + .await + { + Ok(source) => source, + Err(error) => { + tracing::debug!( + error = %error, + attempt, + entity_type = %entity_type, + entity_id = %entity_id, + "field index backfill: stable recovery failed" + ); + continue; + } + }; + let authoritative_sequence = source.durable_sequence(); + let Some(replayed) = source.state.as_ref() else { + outcome = Some(FieldBackfillOutcome::Empty { event_count: 0 }); + break; + }; + last_event_count = replayed.total_event_count as u64; + if replayed.total_event_count == 0 + && source.snapshot.is_none() + && authoritative_sequence == 0 + { + outcome = Some(FieldBackfillOutcome::Empty { + event_count: last_event_count, + }); + break; } - }; - if replayed.total_event_count == 0 { - crate::query_projection_metrics::record_backfill_replay_events( - tenant.as_str(), - entity_type, - "empty", - replayed.total_event_count as u64, - ); - crate::query_projection_metrics::record_backfill_entities( - tenant.as_str(), + match repair_projection_from_stable_source( + state, + tenant, entity_type, - "backfill_replay", - "empty", - 1, - ); - tracing::debug!( - entity_type = %entity_type, - entity_id = %entity_id, - "field index backfill: persistence replay produced no state" - ); - errors += 1; - } else if replayed.status == "Deleted" { - if let Err(e) = query_plane - .remove_projection(tenant.as_str(), entity_type, entity_id) - .await + entity_id, + &store, + &persistence_id, + &source, + ) + .await { - tracing::debug!( - error = %e, + Ok(true) if replayed.status == "Deleted" => { + outcome = Some(FieldBackfillOutcome::Removed { + event_count: last_event_count, + sequence_nr: authoritative_sequence, + }); + break; + } + Ok(true) => { + outcome = Some(FieldBackfillOutcome::Upserted { + event_count: last_event_count, + sequence_nr: authoritative_sequence, + }); + break; + } + Ok(false) => tracing::debug!( + attempt, entity_type = %entity_type, entity_id = %entity_id, - "field index backfill: failed to clear deleted projection" - ); - errors += 1; + "field index backfill: durable source changed during projection repair; retrying" + ), + Err(error) => tracing::debug!( + error = %error, + attempt, + entity_type = %entity_type, + entity_id = %entity_id, + "field index backfill: source-fenced projection repair failed; retrying" + ), + } + } + + match outcome { + Some(FieldBackfillOutcome::Upserted { + event_count, + sequence_nr, + }) => { + indexed += 1; crate::query_projection_metrics::record_backfill_replay_events( tenant.as_str(), entity_type, - "error", - replayed.total_event_count as u64, + "ok", + event_count, ); crate::query_projection_metrics::record_backfill_entities( tenant.as_str(), entity_type, "backfill_replay", - "error", + "ok", 1, ); - } else { + crate::query_projection_metrics::record_update_applied_sequence( + tenant.as_str(), + entity_type, + "upsert", + "backfill_replay", + sequence_nr, + ); + } + Some(FieldBackfillOutcome::Removed { + event_count, + sequence_nr, + }) => { crate::query_projection_metrics::record_backfill_replay_events( tenant.as_str(), entity_type, "deleted", - replayed.total_event_count as u64, + event_count, ); crate::query_projection_metrics::record_backfill_entities( tenant.as_str(), @@ -376,67 +284,40 @@ pub(super) async fn populate_field_index_from_snapshots(state: &ServerState, ten entity_type, "remove", "backfill_replay", - replayed.sequence_nr, + sequence_nr, ); } - } else { - match query_plane - .upsert_projection( + Some(FieldBackfillOutcome::Empty { event_count }) => { + errors += 1; + crate::query_projection_metrics::record_backfill_replay_events( tenant.as_str(), entity_type, - entity_id, - &replayed.status, - &state.query_projection_fields(tenant, entity_type, &replayed.fields), - &state.query_projection_state(&replayed), - replayed.sequence_nr, - ) - .await - { - Ok(()) => { - indexed += 1; - crate::query_projection_metrics::record_backfill_replay_events( - tenant.as_str(), - entity_type, - "ok", - replayed.total_event_count as u64, - ); - crate::query_projection_metrics::record_backfill_entities( - tenant.as_str(), - entity_type, - "backfill_replay", - "ok", - 1, - ); - crate::query_projection_metrics::record_update_applied_sequence( - tenant.as_str(), - entity_type, - "upsert", - "backfill_replay", - replayed.sequence_nr, - ); - } - Err(e) => { - tracing::debug!( - error = %e, - entity_type = %entity_type, - entity_id = %entity_id, - "field index backfill: replay upsert failed" - ); - errors += 1; - crate::query_projection_metrics::record_backfill_replay_events( - tenant.as_str(), - entity_type, - "error", - replayed.total_event_count as u64, - ); - crate::query_projection_metrics::record_backfill_entities( - tenant.as_str(), - entity_type, - "backfill_replay", - "error", - 1, - ); - } + "empty", + event_count, + ); + crate::query_projection_metrics::record_backfill_entities( + tenant.as_str(), + entity_type, + "backfill_replay", + "empty", + 1, + ); + } + None => { + errors += 1; + crate::query_projection_metrics::record_backfill_replay_events( + tenant.as_str(), + entity_type, + "error", + last_event_count, + ); + crate::query_projection_metrics::record_backfill_entities( + tenant.as_str(), + entity_type, + "backfill_replay", + "error", + 1, + ); } } @@ -448,7 +329,7 @@ pub(super) async fn populate_field_index_from_snapshots(state: &ServerState, ten total, indexed, errors, - "populated query projections (snapshots + persistence replay)" + "populated query projections from stable snapshot/journal recovery" ); crate::query_projection_metrics::record_backfill_duration( tenant.as_str(), diff --git a/crates/temper-server/src/state/projection_backfill/entity_load.rs b/crates/temper-server/src/state/projection_backfill/entity_load.rs new file mode 100644 index 000000000..06aff36a7 --- /dev/null +++ b/crates/temper-server/src/state/projection_backfill/entity_load.rs @@ -0,0 +1,269 @@ +//! Durable source selection for projection backfill. + +use temper_runtime::tenant::TenantId; + +use crate::entity_actor::{ + CapturedEntitySnapshot, EntityRecoveryContext, StableEntitySource, + recover_entity_state_from_stable_sources, +}; + +/// Outcome of loading one entity's current state for an index backfill (ADR-0153, +/// ADR-0155). Shared by the key and vector backfills so they classify entities the +/// same way — the distinction is the watermark soundness gate. +pub(super) enum EntityLoadOutcome { + /// Loaded — index it from these fields, fenced at the replayed journal sequence. + Fields { + fields: serde_json::Value, + sequence_nr: u64, + journal_sequence: u64, + snapshot: Option, + }, + /// Durably deleted. Tombstone replay is authoritative even if an asynchronous + /// catalog row still contains the entity's former live fields. + Deleted { + sequence_nr: u64, + journal_sequence: u64, + snapshot: Option, + }, + /// No replayable events or valid snapshot state. This can be a true key-only + /// phantom, or a migration-era entity whose catalog is its durable state. + Missing { + sequence_nr: u64, + journal_sequence: u64, + snapshot: Option, + }, + /// The entity exists (it was enumerated from the durable store) but its current + /// state could not be loaded — no transition table to replay with, an unreadable + /// snapshot, or a replay error. Indexing it is impossible, so the type must NOT be + /// watermarked; otherwise a read would treat a present-but-unindexed entity as + /// authoritatively covered. This is the soundness gate. + LoadFailed, +} + +/// Load one entity's CURRENT state plus the exact snapshot/journal generation that +/// must remain unchanged until the index row is reconciled. +pub(super) async fn load_entity_current_fields( + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + table: Option<&temper_jit::TransitionTable>, + store: &crate::storage::BoxedEventStore, + backend: crate::storage::BackendLabel, + blob_store: Option<&crate::blob_store::BlobStore>, +) -> EntityLoadOutcome { + let Some(table) = table else { + return EntityLoadOutcome::LoadFailed; + }; + let source: StableEntitySource = + match recover_entity_state_from_stable_sources(EntityRecoveryContext { + tenant: tenant.as_str(), + entity_type, + entity_id, + table, + store, + backend, + initial_fields: &serde_json::json!({}), + blob_store, + }) + .await + { + Ok(source) => source, + Err(_) => return EntityLoadOutcome::LoadFailed, + }; + let sequence_nr = source.durable_sequence(); + let journal_sequence = source.journal_sequence; + let replayed_state_materialization = source.replayed_state_materialization; + let snapshot = source.snapshot; + match source.state { + None => EntityLoadOutcome::Missing { + sequence_nr, + journal_sequence, + snapshot, + }, + Some(state) if state.status == "Deleted" => EntityLoadOutcome::Deleted { + sequence_nr, + journal_sequence, + snapshot, + }, + Some(state) + if state.total_event_count == 0 + && snapshot.is_none() + && !replayed_state_materialization => + { + EntityLoadOutcome::Missing { + sequence_nr, + journal_sequence, + snapshot, + } + } + Some(state) => EntityLoadOutcome::Fields { + fields: state.fields, + sequence_nr, + journal_sequence, + snapshot, + }, + } +} + +#[cfg(test)] +mod tests { + use std::collections::{BTreeMap, VecDeque}; + + use temper_jit::TransitionTable; + use temper_runtime::persistence::{ + COMPOSITE_EVENT_TYPE, CompositeEvent, EventMetadata, EventStore, PersistenceEnvelope, + }; + use temper_runtime::scheduler::{install_deterministic_context, sim_now, sim_uuid}; + use temper_store_sim::SimEventStore; + + use super::*; + use crate::entity_actor::{EntityState, state_materialization_envelope}; + use crate::storage::{BackendLabel, BoxedEventStore}; + + const KEYED_RECORD_IOA: &str = r#" +[automaton] +name = "KeyedRecord" +states = ["Active"] +initial = "Active" +allow_indefinite_states = ["Active"] + +[[state]] +name = "ExternalId" +type = "string" +initial = "" + +[[key]] +name = "external_id" +properties = ["ExternalId"] +"#; + + fn composite_audit_envelope(persistence_id: &str, entity_id: &str) -> PersistenceEnvelope { + PersistenceEnvelope { + sequence_nr: 0, + event_type: COMPOSITE_EVENT_TYPE.to_string(), + payload: serde_json::to_value(CompositeEvent { + tenant: "default".to_string(), + parent_entity_type: "KeyedRecord".to_string(), + parent_entity_id: entity_id.to_string(), + parent_action: "Audit".to_string(), + composite_idempotency_key: format!("audit-{entity_id}"), + intent_hash: String::new(), + sub_writes: Vec::new(), + }) + .expect("serialize composite audit"), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: sim_now(), + actor_id: persistence_id.to_string(), + }, + } + } + + #[tokio::test] + async fn materialized_zero_domain_event_journal_remains_indexable() { + let (_guard, _clock, _ids) = install_deterministic_context(238); + let store = SimEventStore::no_faults(238); + let boxed = BoxedEventStore::new(store.clone()); + let table = TransitionTable::from_ioa_source(KEYED_RECORD_IOA); + let entity_id = "materialized-key-owner"; + let persistence_id = format!("default:KeyedRecord:{entity_id}"); + let baseline = EntityState { + entity_type: "KeyedRecord".to_string(), + entity_id: entity_id.to_string(), + status: "Active".to_string(), + item_count: 0, + counters: BTreeMap::new(), + booleans: BTreeMap::new(), + lists: BTreeMap::new(), + fields: serde_json::json!({ + "Id": entity_id, + "Status": "Active", + "ExternalId": "durable-owner", + }), + events: VecDeque::new(), + total_event_count: 0, + events_since_snapshot: 0, + last_snapshot_sequence_nr: 0, + sequence_nr: 0, + processed_idempotency_keys: BTreeMap::new(), + }; + let materialization = state_materialization_envelope(&persistence_id, &baseline, sim_now()) + .expect("serialize state materialization"); + store + .append( + &persistence_id, + 0, + &[ + materialization, + composite_audit_envelope(&persistence_id, entity_id), + ], + ) + .await + .expect("persist materialization and audit"); + + let loaded = load_entity_current_fields( + &TenantId::default(), + "KeyedRecord", + entity_id, + Some(&table), + &boxed, + BackendLabel::Sim, + None, + ) + .await; + + match loaded { + EntityLoadOutcome::Fields { + fields, + sequence_nr, + journal_sequence, + snapshot, + } => { + assert_eq!(fields["ExternalId"], "durable-owner"); + assert_eq!(sequence_nr, 2); + assert_eq!(journal_sequence, 2); + assert!(snapshot.is_none()); + } + _ => panic!( + "a valid state materialization must distinguish an entity from an audit-only phantom" + ), + } + } + + #[tokio::test] + async fn audit_only_zero_domain_event_journal_remains_missing() { + let (_guard, _clock, _ids) = install_deterministic_context(239); + let store = SimEventStore::no_faults(239); + let boxed = BoxedEventStore::new(store.clone()); + let table = TransitionTable::from_ioa_source(KEYED_RECORD_IOA); + let entity_id = "audit-only-phantom"; + let persistence_id = format!("default:KeyedRecord:{entity_id}"); + store + .append( + &persistence_id, + 0, + &[composite_audit_envelope(&persistence_id, entity_id)], + ) + .await + .expect("persist audit-only journal"); + + assert!(matches!( + load_entity_current_fields( + &TenantId::default(), + "KeyedRecord", + entity_id, + Some(&table), + &boxed, + BackendLabel::Sim, + None, + ) + .await, + EntityLoadOutcome::Missing { + journal_sequence: 1, + .. + } + )); + } +} diff --git a/crates/temper-server/src/state/projection_backfill/key_index.rs b/crates/temper-server/src/state/projection_backfill/key_index.rs index 22b47a405..1ed924282 100644 --- a/crates/temper-server/src/state/projection_backfill/key_index.rs +++ b/crates/temper-server/src/state/projection_backfill/key_index.rs @@ -1,214 +1,413 @@ //! ADR-0153 declared-key backfill: key `entity_key_index` for pre-existing entities -//! and record the per-(tenant, entity_type) watermark, so a keyed read MISS can mean -//! authoritative absence (retiring #324's full-type scan — the 413, ARN-68). - -use std::collections::BTreeSet; +//! and record the per-(tenant, entity_type) watermark, so keyed hits and misses can +//! be authoritative (retiring #324's full-type scan — the 413, ARN-68). use temper_runtime::tenant::TenantId; use crate::ServerState; -use super::{EntityLoadOutcome, load_entity_current_fields, transition_table_for}; - -/// Backfill `entity_key_index` for existing entities, then record the watermark. -/// -/// Enumeration is authoritative: keyed types come from the registry and their entity -/// ids from `store.list_entity_ids_by_type`. It must NOT read `state.entity_index`, -/// which is populated only when an actor spawns (lazy) and is therefore near-empty at -/// boot — the original bug that left ~0 of N entities keyed. -/// -/// Robustness at scale (tenants hold 10k–100k+ entities of a keyed type): -/// - **Resumable**: already-keyed entities are skipped (the costly step is loading -/// each entity's state), so a re-run after a partial pass only processes the -/// remainder instead of re-loading all N. -/// - **Sound**: a type is watermarked only if EVERY existing entity was either keyed -/// or is definitively skippable (deleted/phantom). One entity that exists but -/// cannot be loaded fails the type — it is not watermarked, and keyed misses keep -/// scanning (correct, just not bounded) until the data issue is resolved. The -/// failing `entity_id`s are logged so the integrity problem is visible. -/// - **Cooperative**: yields between entities and runs as a background task, so the -/// heavy types (e.g. SessionEntry) do not block boot or the smaller, higher-value -/// types. Types are processed in the registry's sorted order. -/// -/// Idempotent; entities written after the key was declared already co-commit their -/// keys at write time. +use super::{EntityLoadOutcome, load_entity_current_fields}; + +mod activation; + +use activation::prepare_key_index_coverage_for_tables; +pub(in crate::state) use activation::{ + PreparedKeyIndexCoverage, prepare_key_index_coverage_for_activation, + publish_prepared_key_index_coverage, +}; + +enum KeyRepairDisposition { + Reconcile { + fields: Option, + sequence_nr: u64, + journal_sequence: u64, + snapshot: Option, + }, + Unresolved, +} + +fn loaded_sequence_nr(outcome: &EntityLoadOutcome) -> Option { + match outcome { + EntityLoadOutcome::Fields { sequence_nr, .. } + | EntityLoadOutcome::Deleted { sequence_nr, .. } + | EntityLoadOutcome::Missing { sequence_nr, .. } => Some(*sequence_nr), + EntityLoadOutcome::LoadFailed => None, + } +} + +fn loaded_journal_sequence(outcome: &EntityLoadOutcome) -> Option { + match outcome { + EntityLoadOutcome::Fields { + journal_sequence, .. + } + | EntityLoadOutcome::Deleted { + journal_sequence, .. + } + | EntityLoadOutcome::Missing { + journal_sequence, .. + } => Some(*journal_sequence), + EntityLoadOutcome::LoadFailed => None, + } +} + +fn loaded_snapshot( + outcome: &EntityLoadOutcome, +) -> Option<&crate::entity_actor::CapturedEntitySnapshot> { + match outcome { + EntityLoadOutcome::Fields { snapshot, .. } + | EntityLoadOutcome::Deleted { snapshot, .. } + | EntityLoadOutcome::Missing { snapshot, .. } => snapshot.as_ref(), + EntityLoadOutcome::LoadFailed => None, + } +} + +fn reconciliation_sequence( + outcome: &EntityLoadOutcome, + catalog_sequence: Option, +) -> Option { + let journal_sequence = loaded_journal_sequence(outcome)?; + if journal_sequence > 0 { + return Some(journal_sequence); + } + if let Some(snapshot) = loaded_snapshot(outcome) { + return Some(snapshot.sequence_nr); + } + catalog_sequence.or_else(|| loaded_sequence_nr(outcome)) +} + +/// Backfill current registry tables and publish each successful watermark. pub(in crate::state) async fn populate_key_index_from_snapshots( state: &ServerState, tenant: &TenantId, ) { - let Some((store, backend)) = state.event_journal() else { - return; - }; - - // Keyed entity types from the registry — the authoritative record of what is - // installed for this tenant (os-app entities live here, not in transition_tables). - let keyed_types: Vec<(String, Vec)> = { - let registry = state.registry.read().unwrap(); + let tables = { + let registry = state.registry.read().expect("registry lock poisoned"); registry .entity_types(tenant) .into_iter() .filter_map(|entity_type| { - let table = registry.get_table(tenant, entity_type)?; - if table.keys.is_empty() { - None - } else { - Some((entity_type.to_string(), table.keys.clone())) - } + registry + .get_table(tenant, entity_type) + .map(|table| (entity_type.to_string(), table.as_ref().clone())) }) - .collect() + .collect::>() }; + for table in tables { + match prepare_key_index_coverage_for_tables(state, tenant, std::slice::from_ref(&table)) + .await + { + Ok(prepared) => { + if let Err(error) = + publish_prepared_key_index_coverage(state, tenant, &prepared).await + { + tracing::warn!(tenant = %tenant, error = %error, "key index backfill did not publish coverage"); + } + } + Err(error) => tracing::warn!( + tenant = %tenant, + entity_type = %table.0, + error = %error, + "key index backfill did not prepare coverage" + ), + } + } +} - for (entity_type, keys) in &keyed_types { - let current_key_set = crate::key_index::declared_key_set_signature(keys); - let covered = state +#[expect( + clippy::too_many_arguments, + reason = "key coverage preparation captures one exact tenant/type generation" +)] +async fn prepare_key_index_type( + state: &ServerState, + tenant: &TenantId, + store: &crate::storage::BoxedEventStore, + backend: crate::storage::BackendLabel, + entity_type: &str, + table: &temper_jit::TransitionTable, + preparing_activation: bool, + retained_revision: Option, +) -> Result, String> { + let keys = &table.keys; + let current_key_set = crate::key_index::declared_key_set_signature(keys); + if let Some(revision) = retained_revision { + return Ok(Some(PreparedKeyIndexCoverage { + entity_type: entity_type.to_string(), + key_set: current_key_set, + revision, + total: 0, + newly_keyed: 0, + skipped: 0, + })); + } + let covered = if preparing_activation { + None + } else { + state .key_index_backfill_covered_key_set(tenant, entity_type) - .await; - // Already complete for the CURRENT declared key-set: the co-committed write path - // keeps the index whole, so skip the re-scan. - if covered.as_deref() == Some(current_key_set.as_str()) { - continue; - } - // A watermark that covered a DIFFERENT key-set means a key was declared after - // the first backfill (e.g. Directory `ws_path` added after `name_parent`): - // existing entities are keyed for the old keys but NOT the new one. Since - // `keyed_entity_ids_for_type` is per-entity (any key), the resumability skip - // would wrongly skip them, so force a full re-key that re-loads and re-keys - // every entity under all currently-declared keys (idempotent upsert). - let force_full_rekey = covered.is_some(); - if force_full_rekey { - // One-time on the boot that first sees a changed key-set (incl. the 0011 - // migration, which stamps every existing watermark's key_set to ''). Logged - // so this expected full-reload of the type is distinguishable in Datadog from - // a pathology. - tracing::info!( - tenant = %tenant, entity_type = %entity_type, - covered_key_set = covered.as_deref().unwrap_or(""), - current_key_set = %current_key_set, - "key index backfill: declared key-set changed — re-keying every existing entity of this type (one-time)" - ); - } + .await + }; + if covered.as_deref() == Some(current_key_set.as_str()) { + return Ok(None); + } + tracing::info!( + tenant = %tenant, + entity_type, + covered_key_set = covered.as_deref().unwrap_or(""), + current_key_set = %current_key_set, + "key index backfill: reconciling every durable entity" + ); - let entity_ids = match store - .list_entity_ids_by_type(tenant.as_str(), entity_type) + let repair_revision = store + .begin_key_index_backfill(tenant.as_str(), entity_type, ¤t_key_set) + .await + .map_err(|error| { + format!("failed to establish key repair contract for {tenant}:{entity_type}: {error}") + })?; + let blob_store = state.blob_store_for_tenant(tenant).ok(); + let query_plane = state.query_plane_store(); + let scan_boundary = store + .key_reconciliation_boundary(tenant.as_str(), entity_type) + .await + .map_err(|error| { + format!("failed to capture key repair boundary for {tenant}:{entity_type}: {error}") + })?; + const KEY_REPAIR_PAGE_BUDGET: usize = 256; + let mut cursor = None; + let mut total = 0usize; + let mut newly_keyed = 0usize; + let mut skipped = 0usize; + const FAILED_ENTITY_SAMPLE_BUDGET: usize = 8; + let mut failed_count = 0usize; + let mut failed_sample = Vec::with_capacity(FAILED_ENTITY_SAMPLE_BUDGET); + + while let Some(scan_boundary) = scan_boundary.as_deref() { + let page = store + .list_key_reconciliation_page( + tenant.as_str(), + entity_type, + cursor.as_deref(), + scan_boundary, + KEY_REPAIR_PAGE_BUDGET, + ) .await - { - Ok(ids) => ids, - Err(e) => { - tracing::error!( - tenant = %tenant, entity_type = %entity_type, error = %e, - "key index backfill: failed to enumerate entities; type not watermarked" - ); - continue; - } - }; - - // Resumability: on a FIRST-TIME backfill, skip entities already keyed (avoids - // re-loading their state). On a key-set change we must re-key already-keyed - // entities with the new key, so process all. - let already_keyed: BTreeSet = if force_full_rekey { - BTreeSet::new() - } else { - match store - .keyed_entity_ids_for_type(tenant.as_str(), entity_type) - .await - { - Ok(ids) => ids.into_iter().collect(), - Err(_) => BTreeSet::new(), // cannot resume → process all (correct, slower) - } - }; - - let table = transition_table_for(state, tenant, entity_type); - let blob_store = state.blob_store_for_tenant(tenant).ok(); - let total = entity_ids.len(); - let mut newly_keyed = 0usize; - let mut already = 0usize; - let mut skipped = 0usize; - let mut failed = 0usize; - - for entity_id in &entity_ids { - if already_keyed.contains(entity_id) { - already += 1; - continue; - } - match load_entity_current_fields( + .map_err(|error| { + format!( + "failed to enumerate bounded key repair page for {tenant}:{entity_type}: {error}" + ) + })?; + if page.is_empty() { + break; + } + assert!( + page.len() <= KEY_REPAIR_PAGE_BUDGET, + "key repair store exceeded the requested page budget" + ); + let next_cursor = page + .last() + .expect("non-empty key repair page has a last entity") + .entity_id + .clone(); + total = total.checked_add(page.len()).ok_or_else(|| { + format!("key repair entity count overflow for {tenant}:{entity_type}") + })?; + + for candidate in page { + let entity_id = candidate.entity_id; + let is_live = candidate.is_live; + let loaded = load_entity_current_fields( tenant, entity_type, - entity_id, - table.as_ref(), - &store, + &entity_id, + Some(table), + store, backend, blob_store.as_ref(), ) - .await - { - EntityLoadOutcome::Fields(fields) => { - let Some(field_map) = fields.as_object() else { - skipped += 1; - continue; - }; - let mut key_rows = Vec::new(); - for key in keys { - if let Some(hash) = crate::key_index::canonical_key_hash( - &key.name, - &key.properties, - field_map, - ) { - key_rows.push(temper_runtime::persistence::EntityKeyRow { - key_name: key.name.clone(), - key_hash: hash, - }); + .await; + let disposition = match loaded { + EntityLoadOutcome::Fields { + fields, + sequence_nr, + journal_sequence, + snapshot, + } if is_live => KeyRepairDisposition::Reconcile { + fields: Some(fields), + sequence_nr, + journal_sequence, + snapshot, + }, + EntityLoadOutcome::LoadFailed => KeyRepairDisposition::Unresolved, + EntityLoadOutcome::Missing { + sequence_nr, + journal_sequence, + snapshot, + } if journal_sequence > 0 => KeyRepairDisposition::Reconcile { + fields: None, + sequence_nr, + journal_sequence, + snapshot, + }, + outcome => match loaded_journal_sequence(&outcome) { + None => KeyRepairDisposition::Unresolved, + Some(observed_journal_sequence) => { + let captured_snapshot = loaded_snapshot(&outcome).cloned(); + let catalog_row = if let Some(query_plane) = query_plane.as_ref() { + match query_plane + .load_entity_catalog_rows( + tenant.as_str(), + entity_type, + std::slice::from_ref(&entity_id), + ) + .await + { + Ok(Some(rows)) => { + Ok(rows.into_iter().find(|row| row.entity_id == entity_id)) + } + Ok(None) => Ok(None), + Err(error) => Err(error), + } + } else { + Ok(None) + }; + match catalog_row { + Err(error) => { + tracing::warn!(error = %error, entity_type, entity_id, "key repair catalog fallback failed"); + KeyRepairDisposition::Unresolved + } + Ok(row) if !is_live => match reconciliation_sequence( + &outcome, + row.as_ref().map(|catalog| catalog.sequence_nr), + ) { + Some(sequence_nr) => KeyRepairDisposition::Reconcile { + fields: None, + sequence_nr, + journal_sequence: observed_journal_sequence, + snapshot: captured_snapshot, + }, + None => KeyRepairDisposition::Unresolved, + }, + Ok(row) => { + let sequence_nr = reconciliation_sequence( + &outcome, + row.as_ref().map(|catalog| catalog.sequence_nr), + ) + .expect("loaded outcome has a durable sequence"); + match outcome { + EntityLoadOutcome::Deleted { + journal_sequence, + snapshot, + .. + } => KeyRepairDisposition::Reconcile { + fields: None, + sequence_nr, + journal_sequence, + snapshot, + }, + _ => match row { + Some(row) if row.status == "Deleted" => { + KeyRepairDisposition::Reconcile { + fields: None, + sequence_nr, + journal_sequence: observed_journal_sequence, + snapshot: captured_snapshot, + } + } + Some(row) => KeyRepairDisposition::Reconcile { + fields: Some(row.fields), + sequence_nr, + journal_sequence: observed_journal_sequence, + snapshot: captured_snapshot, + }, + None => KeyRepairDisposition::Unresolved, + }, + } + } } } - if key_rows.is_empty() { - // No resolvable key (all key components absent/null) — the - // entity is not addressable by this key, so skipping is sound. - skipped += 1; - continue; + }, + }; + + match disposition { + KeyRepairDisposition::Reconcile { + fields, + sequence_nr, + journal_sequence, + snapshot, + } => { + let mut key_rows = Vec::new(); + if let Some(field_map) = fields.as_ref().and_then(serde_json::Value::as_object) + { + for key in keys { + if let Some(hash) = crate::key_index::canonical_key_hash( + &key.name, + &key.properties, + field_map, + ) { + key_rows.push(temper_runtime::persistence::EntityKeyRow { + key_name: key.name.clone(), + key_hash: hash, + }); + } + } } match store - .backfill_entity_keys(tenant.as_str(), entity_type, entity_id, &key_rows) + .backfill_entity_keys( + tenant.as_str(), + entity_type, + &entity_id, + sequence_nr, + temper_runtime::persistence::KeyIndexBackfillFence { + key_set_signature: ¤t_key_set, + contract_revision: repair_revision, + expected_journal_sequence: journal_sequence, + expected_entity_live: is_live, + expected_snapshot: snapshot.as_ref().map(|snapshot| { + temper_runtime::persistence::SnapshotBackfillFence { + sequence_nr: snapshot.sequence_nr, + state: &snapshot.state, + } + }), + }, + &key_rows, + ) .await { + Ok(()) if key_rows.is_empty() => skipped += 1, Ok(()) => newly_keyed += 1, - Err(e) => { - failed += 1; - tracing::warn!( - error = %e, entity_type = %entity_type, entity_id = %entity_id, - "key index backfill: upsert failed" - ); + Err(error) => { + tracing::warn!(error = %error, entity_type, entity_id, "key repair write lost its source fence"); + failed_count += 1; + if failed_sample.len() < FAILED_ENTITY_SAMPLE_BUDGET { + failed_sample.push(entity_id.clone()); + } } } } - EntityLoadOutcome::Skip => skipped += 1, - EntityLoadOutcome::LoadFailed => { - failed += 1; - tracing::warn!( - entity_type = %entity_type, entity_id = %entity_id, - "key index backfill: existing entity could not be loaded; type will NOT be watermarked (data integrity issue)" - ); + KeyRepairDisposition::Unresolved => { + failed_count += 1; + if failed_sample.len() < FAILED_ENTITY_SAMPLE_BUDGET { + failed_sample.push(entity_id.clone()); + } } } tokio::task::yield_now().await; } + cursor = Some(next_cursor); + } - // Watermark only if nothing failed — every existing entity was keyed or is - // definitively skippable. Otherwise keyed misses keep scanning (sound), and a - // later boot resumes from the remainder. - if failed == 0 { - state - .mark_key_index_backfilled(tenant, entity_type, ¤t_key_set) - .await; - tracing::info!( - tenant = %tenant, entity_type = %entity_type, key_set = %current_key_set, - total, newly_keyed, already, skipped, - "entity_key_index backfill complete; type watermarked" - ); - } else { - tracing::warn!( - tenant = %tenant, entity_type = %entity_type, - total, newly_keyed, already, skipped, failed, - "key index backfill: {failed} entities unresolved; type NOT watermarked (keyed misses keep scanning; will resume next run)" - ); - } + if failed_count > 0 { + return Err(format!( + "key repair left {} unresolved entities for {tenant}:{entity_type}: {}", + failed_count, + failed_sample.join(",") + )); } + Ok(Some(PreparedKeyIndexCoverage { + entity_type: entity_type.to_string(), + key_set: current_key_set, + revision: repair_revision, + total, + newly_keyed, + skipped, + })) } diff --git a/crates/temper-server/src/state/projection_backfill/key_index/activation.rs b/crates/temper-server/src/state/projection_backfill/key_index/activation.rs new file mode 100644 index 000000000..be911f2cd --- /dev/null +++ b/crates/temper-server/src/state/projection_backfill/key_index/activation.rs @@ -0,0 +1,167 @@ +use std::collections::BTreeMap; + +use temper_runtime::tenant::TenantId; + +use crate::ServerState; + +use super::prepare_key_index_type; + +/// A fully replayed contract whose coverage can be published after the live +/// registry swaps to the corresponding activation epoch. +#[derive(Clone, Debug)] +pub(in crate::state) struct PreparedKeyIndexCoverage { + pub entity_type: String, + pub key_set: String, + pub revision: u64, + pub(super) total: usize, + pub(super) newly_keyed: usize, + pub(super) skipped: usize, +} + +/// Replay candidate tables without publishing their coverage watermark. Hot +/// activation uses this while old-epoch writers are fenced, then publishes the +/// registry and the prepared watermark in that order. +pub(super) async fn prepare_key_index_coverage_for_tables( + state: &ServerState, + tenant: &TenantId, + tables: &[(String, temper_jit::TransitionTable)], +) -> Result, String> { + prepare_key_index_coverage(state, tenant, tables, false).await +} + +/// Prepare activation coverage while reusing a retained durable proof when the +/// key signature and semantic spec fingerprint were unchanged. The activation +/// transaction preserves that watermark but advances the writer epoch; a +/// revision CAS in `publish_prepared_key_index_coverage` still fences a plain +/// append that races after this capture. +pub(in crate::state) async fn prepare_key_index_coverage_for_activation( + state: &ServerState, + tenant: &TenantId, + tables: &[(String, temper_jit::TransitionTable)], +) -> Result, String> { + prepare_key_index_coverage(state, tenant, tables, true).await +} + +async fn prepare_key_index_coverage( + state: &ServerState, + tenant: &TenantId, + tables: &[(String, temper_jit::TransitionTable)], + reuse_durable_coverage: bool, +) -> Result, String> { + let Some((store, backend)) = state.event_journal() else { + return Ok(Vec::new()); + }; + if !store.supports_authoritative_key_index() { + return Ok(Vec::new()); + } + + // Capture revisions before reading watermarks. If a plain writer lands + // between the two reads it removes the watermark, so we replay. If it lands + // after both reads, publication's revision CAS loses and the activation + // retry replays. Reading in the opposite order could pair a removed + // watermark with the writer's newer revision and incorrectly republish it. + let retained_coverage = if reuse_durable_coverage { + let mut revisions = BTreeMap::new(); + for (entity_type, _) in tables { + match store + .key_index_reconciliation_revision(tenant.as_str(), entity_type) + .await + { + Ok(revision) => { + revisions.insert(entity_type.clone(), revision); + } + Err(error) => tracing::warn!( + tenant = %tenant, + entity_type, + error = %error, + "could not capture retained key coverage revision; replaying type" + ), + } + } + let watermarks = match store.key_index_backfilled_types(tenant.as_str()).await { + Ok(watermarks) => watermarks.into_iter().collect::>(), + Err(error) => { + tracing::warn!( + tenant = %tenant, + error = %error, + "could not read retained key coverage; replaying activation" + ); + BTreeMap::new() + } + }; + Some((revisions, watermarks)) + } else { + None + }; + + let mut prepared = Vec::new(); + for (entity_type, table) in tables { + let current_key_set = crate::key_index::declared_key_set_signature(&table.keys); + let retained_revision = retained_coverage + .as_ref() + .and_then(|(revisions, watermarks)| { + (watermarks.get(entity_type) == Some(¤t_key_set)) + .then(|| revisions.get(entity_type).copied()) + .flatten() + }); + if let Some(contract) = prepare_key_index_type( + state, + tenant, + &store, + backend, + entity_type, + table, + reuse_durable_coverage, + retained_revision, + ) + .await? + { + prepared.push(contract); + } + } + Ok(prepared) +} + +/// Publish prepared coverage with a revision CAS. Returning `false` as an error +/// keeps activation not-ready so no new-epoch live writer can enter on stale rows. +pub(in crate::state) async fn publish_prepared_key_index_coverage( + state: &ServerState, + tenant: &TenantId, + prepared: &[PreparedKeyIndexCoverage], +) -> Result<(), String> { + for contract in prepared { + match state + .mark_key_index_backfilled_if_revision( + tenant, + &contract.entity_type, + &contract.key_set, + contract.revision, + ) + .await + { + Ok(true) => tracing::info!( + tenant = %tenant, + entity_type = %contract.entity_type, + key_set = %contract.key_set, + total = contract.total, + newly_keyed = contract.newly_keyed, + skipped = contract.skipped, + repair_revision = contract.revision, + "entity_key_index activation repair complete; contract ready" + ), + Ok(false) => { + return Err(format!( + "key contract changed after candidate repair for {tenant}:{}", + contract.entity_type + )); + } + Err(error) => { + return Err(format!( + "failed to publish key coverage for {tenant}:{}: {error}", + contract.entity_type + )); + } + } + } + Ok(()) +} diff --git a/crates/temper-server/src/state/projection_backfill/replay_parity.rs b/crates/temper-server/src/state/projection_backfill/replay_parity.rs index efcb9e826..7f2b2d932 100644 --- a/crates/temper-server/src/state/projection_backfill/replay_parity.rs +++ b/crates/temper-server/src/state/projection_backfill/replay_parity.rs @@ -1,111 +1,18 @@ use std::collections::BTreeMap; -use std::time::{Duration, Instant}; +use std::time::Instant; use temper_runtime::tenant::TenantId; -use crate::entity_actor::recover_entity_state_from_store; -use crate::state::{ - QueryProjectionReplayParityDrift, QueryProjectionReplayParityReport, ServerState, -}; -use crate::storage::{CatalogRowsLoad, EntityCatalogRow, load_catalog_rows_by_id}; - -const MAX_REPLAY_PARITY_DRIFT_EXAMPLES: usize = 25; +use crate::entity_actor::{EntityRecoveryContext, recover_entity_state_from_stable_sources}; +use crate::state::{QueryProjectionReplayParityReport, ServerState}; +use crate::storage::{CatalogRowsLoad, load_catalog_rows_by_id}; -struct ReplayParityExample<'a> { - entity_type: &'a str, - entity_id: &'a str, - drift_kind: &'a str, - sequence_direction: &'a str, - sequence_gap: u64, - catalog_sequence: Option, - authoritative_sequence: u64, -} +mod metrics; -fn replay_parity_drift_kind( - catalog: &EntityCatalogRow, - authoritative_status: &str, - authoritative_fields: &serde_json::Value, - authoritative_state: &serde_json::Value, - authoritative_sequence: u64, -) -> &'static str { - let status_drift = catalog.status != authoritative_status; - let fields_drift = catalog.fields != *authoritative_fields; - let state_drift = catalog - .state - .as_ref() - .is_some_and(|catalog_state| catalog_state != authoritative_state); - let sequence_drift = catalog.sequence_nr != authoritative_sequence; - match (status_drift, fields_drift, state_drift, sequence_drift) { - (false, false, false, false) => "none", - (true, false, false, false) => "status", - (false, true, false, false) => "fields", - (false, false, true, false) => "state", - (false, false, false, true) => "sequence", - _ => "multiple", - } -} - -fn replay_parity_sequence_gap( - catalog_sequence: Option, - authoritative_sequence: u64, -) -> (&'static str, u64) { - let Some(catalog_sequence) = catalog_sequence else { - return ("catalog_missing", authoritative_sequence); - }; - match catalog_sequence.cmp(&authoritative_sequence) { - std::cmp::Ordering::Less => ("catalog_behind", authoritative_sequence - catalog_sequence), - std::cmp::Ordering::Equal => ("equal", 0), - std::cmp::Ordering::Greater => ("catalog_ahead", catalog_sequence - authoritative_sequence), - } -} - -fn push_replay_parity_example( - report: &mut QueryProjectionReplayParityReport, - example: ReplayParityExample<'_>, -) { - if report.drift_examples.len() >= MAX_REPLAY_PARITY_DRIFT_EXAMPLES { - return; - } - report - .drift_examples - .push(QueryProjectionReplayParityDrift { - entity_type: example.entity_type.to_string(), - entity_id: example.entity_id.to_string(), - drift_kind: example.drift_kind.to_string(), - sequence_direction: example.sequence_direction.to_string(), - sequence_gap: example.sequence_gap, - catalog_sequence: example.catalog_sequence, - authoritative_sequence: example.authoritative_sequence, - }); -} - -#[expect( - clippy::too_many_arguments, - reason = "run-level projection parity metric mirrors the metric dimensions and counters" -)] -fn record_replay_parity_run_summary( - tenant: &TenantId, - entity_type_filter: Option<&str>, - source: &str, - result: &str, - checked: u64, - drifted: u64, - missing: u64, - errors: u64, - duration: Duration, -) { - crate::query_projection_metrics::record_replay_parity_run( - tenant.as_str(), - entity_type_filter.unwrap_or("*"), - source, - result, - checked, - drifted, - missing, - errors, - duration, - ); -} +use metrics::{ + ReplayParityExample, push_replay_parity_example, record_replay_parity_run_summary, + replay_parity_drift_kind, replay_parity_sequence_gap, +}; pub(in crate::state) async fn verify_query_projection_replay_parity( state: &ServerState, @@ -278,20 +185,40 @@ pub(in crate::state) async fn verify_query_projection_replay_parity( let started_at = Instant::now(); // determinism-ok: production-only parity verifier duration metric report.checked += 1; let tenant_blob_store = state.blob_store_for_tenant(tenant).ok(); - let replayed = recover_entity_state_from_store( - tenant.as_str(), - &entity_type, - &entity_id, - &table, - &store, + let replayed = recover_entity_state_from_stable_sources(EntityRecoveryContext { + tenant: tenant.as_str(), + entity_type: &entity_type, + entity_id: &entity_id, + table: &table, + store: &store, backend, - &serde_json::json!({}), - tenant_blob_store.as_ref(), - false, // replay-parity check: lenient (unchanged behavior) - ) + initial_fields: &serde_json::json!({}), + blob_store: tenant_blob_store.as_ref(), + }) .await; - let replayed = match replayed { - Ok(state) => state, + let (replayed, authoritative_sequence) = match replayed { + Ok(source) => { + let authoritative_sequence = source.durable_sequence(); + let Some(state) = source.state else { + report.errors += 1; + push_replay_parity_example( + &mut report, + ReplayParityExample { + entity_type: &entity_type, + entity_id: &entity_id, + drift_kind: "replay_absent", + sequence_direction: "unknown", + sequence_gap: 0, + catalog_sequence: catalog_rows + .get(&entity_id) + .map(|catalog| catalog.sequence_nr), + authoritative_sequence, + }, + ); + continue; + }; + (state, authoritative_sequence) + } Err(error) => { report.errors += 1; push_replay_parity_example( @@ -332,8 +259,10 @@ pub(in crate::state) async fn verify_query_projection_replay_parity( if replayed.status == "Deleted" { if let Some(catalog) = catalog { report.drifted += 1; - let (sequence_direction, sequence_gap) = - replay_parity_sequence_gap(Some(catalog.sequence_nr), replayed.sequence_nr); + let (sequence_direction, sequence_gap) = replay_parity_sequence_gap( + Some(catalog.sequence_nr), + authoritative_sequence, + ); push_replay_parity_example( &mut report, ReplayParityExample { @@ -343,7 +272,7 @@ pub(in crate::state) async fn verify_query_projection_replay_parity( sequence_direction, sequence_gap, catalog_sequence: Some(catalog.sequence_nr), - authoritative_sequence: replayed.sequence_nr, + authoritative_sequence, }, ); crate::query_projection_metrics::record_replay_parity_check( @@ -374,7 +303,7 @@ pub(in crate::state) async fn verify_query_projection_replay_parity( report.missing += 1; report.drifted += 1; let (sequence_direction, sequence_gap) = - replay_parity_sequence_gap(None, replayed.sequence_nr); + replay_parity_sequence_gap(None, authoritative_sequence); push_replay_parity_example( &mut report, ReplayParityExample { @@ -384,7 +313,7 @@ pub(in crate::state) async fn verify_query_projection_replay_parity( sequence_direction, sequence_gap, catalog_sequence: None, - authoritative_sequence: replayed.sequence_nr, + authoritative_sequence, }, ); crate::query_projection_metrics::record_replay_parity_check( @@ -407,10 +336,10 @@ pub(in crate::state) async fn verify_query_projection_replay_parity( &replayed.status, &projected_fields, &projected_state, - replayed.sequence_nr, + authoritative_sequence, ); let (sequence_direction, sequence_gap) = - replay_parity_sequence_gap(Some(catalog.sequence_nr), replayed.sequence_nr); + replay_parity_sequence_gap(Some(catalog.sequence_nr), authoritative_sequence); if drift_kind == "none" { report.matched += 1; crate::query_projection_metrics::record_replay_parity_check( @@ -433,7 +362,7 @@ pub(in crate::state) async fn verify_query_projection_replay_parity( sequence_direction, sequence_gap, catalog_sequence: Some(catalog.sequence_nr), - authoritative_sequence: replayed.sequence_nr, + authoritative_sequence, }, ); crate::query_projection_metrics::record_replay_parity_check( diff --git a/crates/temper-server/src/state/projection_backfill/replay_parity/metrics.rs b/crates/temper-server/src/state/projection_backfill/replay_parity/metrics.rs new file mode 100644 index 000000000..e91f5f985 --- /dev/null +++ b/crates/temper-server/src/state/projection_backfill/replay_parity/metrics.rs @@ -0,0 +1,104 @@ +use std::time::Duration; + +use temper_runtime::tenant::TenantId; + +use crate::state::{QueryProjectionReplayParityDrift, QueryProjectionReplayParityReport}; +use crate::storage::EntityCatalogRow; + +const MAX_REPLAY_PARITY_DRIFT_EXAMPLES: usize = 25; + +pub(super) struct ReplayParityExample<'a> { + pub(super) entity_type: &'a str, + pub(super) entity_id: &'a str, + pub(super) drift_kind: &'a str, + pub(super) sequence_direction: &'a str, + pub(super) sequence_gap: u64, + pub(super) catalog_sequence: Option, + pub(super) authoritative_sequence: u64, +} + +pub(super) fn replay_parity_drift_kind( + catalog: &EntityCatalogRow, + authoritative_status: &str, + authoritative_fields: &serde_json::Value, + authoritative_state: &serde_json::Value, + authoritative_sequence: u64, +) -> &'static str { + let status_drift = catalog.status != authoritative_status; + let fields_drift = catalog.fields != *authoritative_fields; + let state_drift = catalog + .state + .as_ref() + .is_some_and(|catalog_state| catalog_state != authoritative_state); + let sequence_drift = catalog.sequence_nr != authoritative_sequence; + match (status_drift, fields_drift, state_drift, sequence_drift) { + (false, false, false, false) => "none", + (true, false, false, false) => "status", + (false, true, false, false) => "fields", + (false, false, true, false) => "state", + (false, false, false, true) => "sequence", + _ => "multiple", + } +} + +pub(super) fn replay_parity_sequence_gap( + catalog_sequence: Option, + authoritative_sequence: u64, +) -> (&'static str, u64) { + let Some(catalog_sequence) = catalog_sequence else { + return ("catalog_missing", authoritative_sequence); + }; + match catalog_sequence.cmp(&authoritative_sequence) { + std::cmp::Ordering::Less => ("catalog_behind", authoritative_sequence - catalog_sequence), + std::cmp::Ordering::Equal => ("equal", 0), + std::cmp::Ordering::Greater => ("catalog_ahead", catalog_sequence - authoritative_sequence), + } +} + +pub(super) fn push_replay_parity_example( + report: &mut QueryProjectionReplayParityReport, + example: ReplayParityExample<'_>, +) { + if report.drift_examples.len() >= MAX_REPLAY_PARITY_DRIFT_EXAMPLES { + return; + } + report + .drift_examples + .push(QueryProjectionReplayParityDrift { + entity_type: example.entity_type.to_string(), + entity_id: example.entity_id.to_string(), + drift_kind: example.drift_kind.to_string(), + sequence_direction: example.sequence_direction.to_string(), + sequence_gap: example.sequence_gap, + catalog_sequence: example.catalog_sequence, + authoritative_sequence: example.authoritative_sequence, + }); +} + +#[expect( + clippy::too_many_arguments, + reason = "run-level projection parity metric mirrors the metric dimensions and counters" +)] +pub(super) fn record_replay_parity_run_summary( + tenant: &TenantId, + entity_type_filter: Option<&str>, + source: &str, + result: &str, + checked: u64, + drifted: u64, + missing: u64, + errors: u64, + duration: Duration, +) { + crate::query_projection_metrics::record_replay_parity_run( + tenant.as_str(), + entity_type_filter.unwrap_or("*"), + source, + result, + checked, + drifted, + missing, + errors, + duration, + ); +} diff --git a/crates/temper-server/src/state/projection_backfill/vector_index.rs b/crates/temper-server/src/state/projection_backfill/vector_index.rs index 44d0de6fe..b573553fe 100644 --- a/crates/temper-server/src/state/projection_backfill/vector_index.rs +++ b/crates/temper-server/src/state/projection_backfill/vector_index.rs @@ -128,7 +128,7 @@ pub(in crate::state) async fn populate_vector_index_from_snapshots( ) .await { - EntityLoadOutcome::Fields(fields) => { + EntityLoadOutcome::Fields { fields, .. } => { let Some(field_map) = fields.as_object() else { skipped += 1; continue; @@ -179,7 +179,7 @@ pub(in crate::state) async fn populate_vector_index_from_snapshots( } } } - EntityLoadOutcome::Skip => { + EntityLoadOutcome::Deleted { .. } | EntityLoadOutcome::Missing { .. } => { // A deleted (or phantom) entity must hold no vector rows — purge // any it still has so a soft-deleted entity is never ranked // (reconcile with an empty row set). Harmless when there is nothing diff --git a/crates/temper-server/src/state/source_fenced_projection.rs b/crates/temper-server/src/state/source_fenced_projection.rs new file mode 100644 index 000000000..b7c9493db --- /dev/null +++ b/crates/temper-server/src/state/source_fenced_projection.rs @@ -0,0 +1,241 @@ +//! Exact-source reconciliation for derived query projections. + +use temper_runtime::actor::ActorError; +use temper_runtime::persistence::{ProjectionSourceFence, SnapshotBackfillFence}; +use temper_runtime::tenant::TenantId; + +use crate::entity_actor::{ + EntityRecoveryContext, StableEntitySource, recover_entity_state_from_stable_sources, + stable_entity_source_is_current, +}; +use crate::storage::BoxedEventStore; + +use super::ServerState; + +const MAX_DIRTY_PROJECTION_REPAIR_ATTEMPTS: usize = 3; + +/// Reconcile one recovered entity projection while its exact durable source is +/// current, then close the source fence. `false` means the caller must recover +/// and retry from a new generation. +pub(crate) async fn repair_projection_from_stable_source( + state: &ServerState, + tenant: &TenantId, + entity_type: &str, + entity_id: &str, + store: &BoxedEventStore, + persistence_id: &str, + source: &StableEntitySource, +) -> Result { + let Some(query_plane) = state.query_plane_store() else { + return stable_entity_source_is_current(store, persistence_id, source).await; + }; + let source_fence = ProjectionSourceFence { + expected_journal_sequence: source.journal_sequence, + expected_snapshot: source + .snapshot + .as_ref() + .map(|snapshot| SnapshotBackfillFence { + sequence_nr: snapshot.sequence_nr, + state: snapshot.state.as_slice(), + }), + }; + + let Some(entity_state) = source.state.as_ref() else { + let applied = query_plane + .clear_projection_dirty_if_source( + tenant.as_str(), + entity_type, + entity_id, + source_fence, + ) + .await + .map_err(|error| { + ActorError::custom(format!( + "source-fenced empty projection acknowledgement failed for {entity_type}:{entity_id}: {error}" + )) + })?; + if !applied { + return Ok(false); + } + return stable_entity_source_is_current(store, persistence_id, source).await; + }; + + if entity_state.status == "Deleted" { + let applied = query_plane + .remove_projection_if_source(tenant.as_str(), entity_type, entity_id, source_fence) + .await + .map_err(|error| { + ActorError::custom(format!( + "source-fenced projection removal failed for {entity_type}:{entity_id}: {error}" + )) + })?; + if !applied { + return Ok(false); + } + return stable_entity_source_is_current(store, persistence_id, source).await; + } + + let fields = state.query_projection_fields(tenant, entity_type, &entity_state.fields); + let projected_state = state.query_projection_state(entity_state); + let sequence_nr = source.durable_sequence(); + let applied = query_plane + .upsert_projection_if_source( + tenant.as_str(), + entity_type, + entity_id, + &entity_state.status, + &fields, + &projected_state, + sequence_nr, + source_fence, + ) + .await + .map_err(|error| { + ActorError::custom(format!( + "source-fenced projection upsert failed for {entity_type}:{entity_id}: {error}" + )) + })?; + if !applied { + return Ok(false); + } + + match stable_entity_source_is_current(store, persistence_id, source).await { + Ok(true) => Ok(true), + closing_result => { + // The source advanced after the conditional write committed. Remove + // only that exact attempted catalog row; a concurrent newer row with + // the same sequence but different full state is preserved. Projection + // absence is fail-closed because query reads detect the field-index + // coverage gap and reconcile from authoritative state. + query_plane + .remove_projection_if_exact( + tenant.as_str(), + entity_type, + entity_id, + &entity_state.status, + &fields, + &projected_state, + sequence_nr, + ) + .await + .map_err(|cleanup_error| { + ActorError::custom(format!( + "failed to clean unstable projection for {entity_type}:{entity_id}: {cleanup_error}" + )) + })?; + match closing_result { + Ok(false) => Ok(false), + Err(error) => Err(error), + Ok(true) => unreachable!("handled above"), + } + } + } +} + +/// Repair every durably dirty projection for one entity type before a native +/// query-plane read can trust the catalog or field index. Source writers mark +/// dirty in the same transaction as their journal/snapshot mutation; a +/// source-fenced projection mutation clears it in the same backend transaction. +/// Exhaustion leaves the marker durable so the read fails closed and retries. +pub(crate) async fn repair_dirty_projections_before_read( + state: &ServerState, + tenant: &TenantId, + entity_type: &str, + repair_budget: usize, +) -> Result<(), ActorError> { + let repair_budget = repair_budget.max(1); + let Some(query_plane) = state.query_plane_store() else { + return Ok(()); + }; + let dirty_ids = query_plane + .dirty_projection_entity_ids( + tenant.as_str(), + entity_type, + repair_budget.saturating_add(1), + ) + .await + .map_err(|error| { + ActorError::custom(format!( + "failed to enumerate dirty query projections for {entity_type}: {error}" + )) + })?; + let Some(mut dirty_ids) = dirty_ids else { + return Ok(()); + }; + if dirty_ids.is_empty() { + return Ok(()); + } + let exceeded_budget = dirty_ids.len() > repair_budget; + dirty_ids.truncate(repair_budget); + + let Some((store, backend)) = state.event_journal() else { + return Err(ActorError::custom(format!( + "dirty query projections for {entity_type} cannot be repaired without an event journal" + ))); + }; + let Some(table) = super::projection_backfill::transition_table_for(state, tenant, entity_type) + else { + return Err(ActorError::custom(format!( + "dirty query projections for {entity_type} cannot be repaired without a transition table" + ))); + }; + let tenant_blob_store = state.blob_store_for_tenant(tenant).ok(); + let initial_fields = serde_json::json!({}); + + for entity_id in dirty_ids { + let persistence_id = format!("{tenant}:{entity_type}:{entity_id}"); + let mut repaired = false; + for _attempt in 1..=MAX_DIRTY_PROJECTION_REPAIR_ATTEMPTS { + let source = recover_entity_state_from_stable_sources(EntityRecoveryContext { + tenant: tenant.as_str(), + entity_type, + entity_id: &entity_id, + table: &table, + store: &store, + backend, + initial_fields: &initial_fields, + blob_store: tenant_blob_store.as_ref(), + }) + .await?; + if repair_projection_from_stable_source( + state, + tenant, + entity_type, + &entity_id, + &store, + &persistence_id, + &source, + ) + .await? + { + repaired = true; + break; + } + } + if !repaired { + return Err(ActorError::custom(format!( + "dirty query projection for {entity_type}:{entity_id} did not stabilize after {MAX_DIRTY_PROJECTION_REPAIR_ATTEMPTS} attempts" + ))); + } + } + + let remaining = query_plane + .dirty_projection_entity_ids(tenant.as_str(), entity_type, 1) + .await + .map_err(|error| { + ActorError::custom(format!( + "failed to close dirty query projection repair for {entity_type}: {error}" + )) + })?; + if remaining.as_ref().is_some_and(|ids| !ids.is_empty()) { + return Err(ActorError::custom(format!( + "dirty query projections remain for {entity_type} after bounded repair of {repair_budget} entities{}", + if exceeded_budget { + " (initial batch exceeded the repair budget)" + } else { + "" + } + ))); + } + Ok(()) +} diff --git a/crates/temper-server/src/state/tenant_generation.rs b/crates/temper-server/src/state/tenant_generation.rs new file mode 100644 index 000000000..52100fb26 --- /dev/null +++ b/crates/temper-server/src/state/tenant_generation.rs @@ -0,0 +1,440 @@ +//! Tenant runtime-generation publication guards and request admission. + +use super::*; + +mod lease; +pub use lease::TenantGenerationLease; + +/// Proof that one tenant's durable/runtime spec publication is serialized in +/// Temper's single-process runtime. +/// +/// Callers acquire this before durable persistence and retain it until registry +/// publication and key-contract readiness both complete. Once armed, dropping +/// the guard deliberately does not reopen traffic: a storage error can be +/// outcome-ambiguous, so only a completed cutover may release the tenant gate. +pub struct SpecPublicationGuard { + pub(in crate::state) tenant: String, + pub(in crate::state) gated_tenants: Arc>>, + pub(in crate::state) publication_debts: Arc>>, + pub(in crate::state) generation_versions: Arc>>, + pub(in crate::state) acquired_generation: u64, + pub(in crate::state) intent_fingerprint: Option, + pub(in crate::state) inherited_debt: bool, + pub(in crate::state) armed: bool, + pub(in crate::state) _guard: tokio::sync::OwnedRwLockWriteGuard<()>, +} + +impl SpecPublicationGuard { + pub(in crate::state) fn validates(&self, tenant: &TenantId) -> Result<(), String> { + if self.tenant != tenant.as_str() { + return Err(format!( + "spec publication guard for tenant {} cannot publish tenant {tenant}", + self.tenant + )); + } + if !self.armed { + return Err(format!( + "spec publication guard for tenant {tenant} was not armed before persistence" + )); + } + Ok(()) + } + + pub(in crate::state) fn arm( + &mut self, + tenant: &TenantId, + intent_fingerprint: &str, + ) -> Result<(), String> { + if self.tenant != tenant.as_str() { + return Err(format!( + "spec publication guard for tenant {} cannot arm tenant {tenant}", + self.tenant + )); + } + if intent_fingerprint.is_empty() { + return Err(format!( + "spec publication intent for tenant {tenant} must not be empty" + )); + } + if self.armed { + return Err(format!( + "spec publication guard for tenant {tenant} is already armed" + )); + } + let mut debts = self + .publication_debts + .write() + .map_err(|error| format!("spec publication debt lock poisoned: {error}"))?; + self.inherited_debt = match debts.get(tenant.as_str()) { + Some(existing) if existing != intent_fingerprint => { + return Err(format!( + "tenant {tenant} has an unresolved spec publication; retry its exact runtime generation before publishing a different one" + )); + } + Some(_) => true, + None => false, + }; + self.gated_tenants + .write() + .map_err(|error| format!("spec publication gate lock poisoned: {error}"))? + .insert(self.tenant.clone()); + debts.insert(self.tenant.clone(), intent_fingerprint.to_string()); + self.intent_fingerprint = Some(intent_fingerprint.to_string()); + self.armed = true; + Ok(()) + } + + pub(in crate::state) fn release(&mut self, allow_inherited_debt: bool) -> Result<(), String> { + if self.inherited_debt && !allow_inherited_debt { + return Err(format!( + "tenant {} requires its complete publication workflow to discharge the inherited runtime-generation debt", + self.tenant + )); + } + let intent_fingerprint = self.intent_fingerprint.as_deref().ok_or_else(|| { + format!( + "spec publication guard for tenant {} has no armed intent", + self.tenant + ) + })?; + let mut debts = self + .publication_debts + .write() + .map_err(|error| format!("spec publication debt lock poisoned: {error}"))?; + if debts.get(&self.tenant).map(String::as_str) != Some(intent_fingerprint) { + return Err(format!( + "spec publication intent changed before tenant {} completed", + self.tenant + )); + } + self.gated_tenants + .write() + .map_err(|error| format!("spec publication gate lock poisoned: {error}"))? + .remove(&self.tenant); + debts.remove(&self.tenant); + let mut versions = self + .generation_versions + .write() + .expect("tenant generation version lock poisoned"); + let version = versions.entry(self.tenant.clone()).or_insert(0); + *version = version + .checked_add(1) + .expect("tenant runtime generation counter overflowed"); + self.acquired_generation = *version; + self.intent_fingerprint = None; + self.inherited_debt = false; + self.armed = false; + Ok(()) + } +} + +#[allow(deprecated)] // ADR-0025 Phase 4: RecordStore used until chain validation replaced +impl ServerState { + /// Try to acquire the tenant coordinator before any durable spec publication + /// and retain it through registry publication plus key-contract readiness. + /// + /// The coordinator is intentionally process-local because the current actor + /// runtime supports exactly one server process. A distributed runtime must + /// replace this with a durable lease spanning the full cutover. + pub async fn begin_spec_publication( + &self, + tenant: &TenantId, + ) -> Result { + let tenant_lock = { + let mut locks = self.spec_publication_locks.lock().await; + locks + .entry(tenant.as_str().to_string()) + .or_insert_with(|| Arc::new(tokio::sync::RwLock::new(()))) + .clone() + }; + let guard = tenant_lock.try_write_owned().map_err(|_| { + format!("tenant {tenant} runtime generation is busy; retry spec publication") + })?; + let acquired_generation = self.tenant_generation_version(tenant); + Ok(SpecPublicationGuard { + tenant: tenant.as_str().to_string(), + gated_tenants: Arc::clone(&self.spec_publication_gated_tenants), + publication_debts: Arc::clone(&self.spec_publication_debts), + generation_versions: Arc::clone(&self.tenant_generation_versions), + acquired_generation, + intent_fingerprint: None, + inherited_debt: false, + armed: false, + _guard: guard, + }) + } + + /// Acquire a publication writer only if no complete generation committed + /// since a caller released its stable request lease. + pub async fn begin_spec_publication_after( + &self, + tenant: &TenantId, + expected_generation: u64, + ) -> Result { + let guard = self.begin_spec_publication(tenant).await?; + if guard.acquired_generation != expected_generation { + return Err(format!( + "tenant {tenant} advanced from runtime generation {expected_generation} to {} during publication handoff; retry the governed action", + guard.acquired_generation + )); + } + Ok(guard) + } + + /// Drain independently forked work from the captured request generation, + /// then acquire the publication writer if no completed cutover intervened. + /// + /// Generation-handoff hooks use this awaited path after releasing their + /// request lease. A writer therefore cannot overtake post-commit work that + /// was forked before the handoff, and the version check still rejects any + /// different publication that completed while the caller waited. + pub async fn begin_spec_publication_after_drain( + &self, + tenant: &TenantId, + expected_generation: u64, + ) -> Result { + let tenant_lock = { + let mut locks = self.spec_publication_locks.lock().await; + locks + .entry(tenant.as_str().to_string()) + .or_insert_with(|| Arc::new(tokio::sync::RwLock::new(()))) + .clone() + }; + let writer = tenant_lock.write_owned().await; + let acquired_generation = self.tenant_generation_version(tenant); + if acquired_generation != expected_generation { + return Err(format!( + "tenant {tenant} advanced from runtime generation {expected_generation} to {acquired_generation} during publication handoff; retry the governed action" + )); + } + Ok(SpecPublicationGuard { + tenant: tenant.as_str().to_string(), + gated_tenants: Arc::clone(&self.spec_publication_gated_tenants), + publication_debts: Arc::clone(&self.spec_publication_debts), + generation_versions: Arc::clone(&self.tenant_generation_versions), + acquired_generation, + intent_fingerprint: None, + inherited_debt: false, + armed: false, + _guard: writer, + }) + } + + /// Build an internal action context owned by an active publication writer. + /// + /// OS-app content and seed work runs after the new registry generation is + /// installed but before the writer releases external traffic. The released + /// lease is accepted only while this exact tenant remains gated at the + /// writer's captured generation. + pub fn spec_publication_dispatch_context( + &self, + guard: &SpecPublicationGuard, + tenant: &TenantId, + service: &str, + ) -> Result { + if guard.tenant != tenant.as_str() || !guard.armed { + return Err(format!( + "tenant {tenant} has no active publication writer for internal dispatch" + )); + } + if guard.acquired_generation != self.tenant_generation_version(tenant) + || !self.spec_publication_gated(tenant) + { + return Err(format!( + "tenant {tenant} publication generation changed before internal dispatch" + )); + } + Ok( + crate::request_context::AgentContext::for_service(service) + .with_tenant_generation_lease(TenantGenerationLease::publication_owned( + tenant, + guard.acquired_generation, + )), + ) + } + + /// Current complete live generation for one tenant. + pub fn tenant_generation_version(&self, tenant: &TenantId) -> u64 { + self.tenant_generation_versions + .read() + .expect("tenant generation version lock poisoned") + .get(tenant.as_str()) + .copied() + .unwrap_or(0) + } + + /// Canonical digest naming the complete runtime generation a publication + /// intends to make durable and live. Components are sorted by name and + /// length-delimited, so callers can build the same intent on an exact retry + /// without depending on map iteration or concatenation ambiguity. + pub fn spec_publication_intent<'a>( + kind: &str, + components: impl IntoIterator, + ) -> String { + let mut components = components + .into_iter() + .map(|(name, value)| (name.to_string(), value.to_vec())) + .collect::>(); + components.sort_by(|left, right| left.0.cmp(&right.0).then(left.1.cmp(&right.1))); + let mut digest = Sha256::new(); + digest.update((kind.len() as u64).to_be_bytes()); + digest.update(kind.as_bytes()); + for (name, value) in components { + digest.update((name.len() as u64).to_be_bytes()); + digest.update(name.as_bytes()); + digest.update((value.len() as u64).to_be_bytes()); + digest.update(value); + } + format!("{:x}", digest.finalize()) + } + + /// Hold a stable runtime generation while serving one tenant request. + /// A queued publication writer prevents later requests from entering, and + /// waits for earlier requests to finish before any registry/authz/WASM + /// component can change. + pub async fn begin_tenant_request( + &self, + tenant: &TenantId, + ) -> tokio::sync::OwnedRwLockReadGuard<()> { + let tenant_lock = { + let mut locks = self.spec_publication_locks.lock().await; + locks + .entry(tenant.as_str().to_string()) + .or_insert_with(|| Arc::new(tokio::sync::RwLock::new(()))) + .clone() + }; + tenant_lock.read_owned().await + } + + /// Try to enter a stable tenant generation without waiting behind a + /// publication. HTTP entry points use this bounded admission path and + /// return a retryable response while a writer is active or queued. + pub async fn try_begin_tenant_request( + &self, + tenant: &TenantId, + ) -> Option> { + let tenant_lock = { + let mut locks = self.spec_publication_locks.lock().await; + locks + .entry(tenant.as_str().to_string()) + .or_insert_with(|| Arc::new(tokio::sync::RwLock::new(()))) + .clone() + }; + tenant_lock.try_read_owned().ok() + } + + /// Re-enter the exact runtime generation captured by detached work. + /// + /// The second version/gate check runs after acquiring the read side, so a + /// writer can neither slip between validation and work nor let an old task + /// borrow the next generation's registry and policy artifacts. + #[cfg(test)] + pub(crate) async fn try_begin_captured_tenant_generation( + &self, + tenant: &TenantId, + captured_generation: u64, + ) -> Option { + if self.spec_publication_gated(tenant) + || self.tenant_generation_version(tenant) != captured_generation + { + return None; + } + let guard = self.try_begin_tenant_request(tenant).await?; + if self.spec_publication_gated(tenant) + || self.tenant_generation_version(tenant) != captured_generation + { + return None; + } + Some(TenantGenerationLease::new( + tenant, + guard, + captured_generation, + )) + } + + /// Wait behind an active publication, then enter only if its resulting + /// generation is exactly the one expected by queued background work. + /// + /// Publication-owned work uses this to defer effects produced from the + /// pending registry generation until the writer makes that generation + /// externally visible. An ambiguous publication leaves the tenant gated, + /// so queued work fails closed after the writer releases its lock. + pub(crate) async fn begin_captured_tenant_generation( + &self, + tenant: &TenantId, + captured_generation: u64, + ) -> Option { + let guard = self.begin_tenant_request(tenant).await; + if self.spec_publication_gated(tenant) + || self.tenant_generation_version(tenant) != captured_generation + { + return None; + } + Some(TenantGenerationLease::new( + tenant, + guard, + captured_generation, + )) + } + + /// Activate an independently forked context for immediate detached work. + /// Request-owned forks already hold the old generation and therefore drain + /// before publication. Publication-owned tokens wait for the pending + /// generation to become complete. A context without prior generation + /// provenance enters whichever complete generation is current when it runs. + pub(crate) async fn activate_immediate_tenant_work( + &self, + tenant: &TenantId, + mut agent_ctx: crate::request_context::AgentContext, + ) -> Option { + match agent_ctx.tenant_generation_lease.as_ref() { + Some(lease) if lease.is_held_for(tenant) => { + if lease.captured_generation() != self.tenant_generation_version(tenant) { + return None; + } + return Some(agent_ctx); + } + Some(lease) if lease.is_publication_owned_for(tenant) => { + let pending_generation = lease.captured_generation().checked_add(1)?; + let lease = self + .begin_captured_tenant_generation(tenant, pending_generation) + .await?; + agent_ctx.tenant_generation_lease = Some(lease); + return Some(agent_ctx); + } + Some(_) => return None, + None => {} + } + + let guard = self.begin_tenant_request(tenant).await; + if self.spec_publication_gated(tenant) { + return None; + } + let generation = self.tenant_generation_version(tenant); + agent_ctx.tenant_generation_lease = + Some(TenantGenerationLease::new(tenant, guard, generation)); + Some(agent_ctx) + } + + /// Whether an earlier outcome-ambiguous publication still keeps the tenant + /// fail-closed after its writer guard has unwound. + pub fn spec_publication_gated(&self, tenant: &TenantId) -> bool { + self.spec_publication_gated_tenants + .read() + .expect("spec publication gate lock poisoned") + .contains(tenant.as_str()) + } + + pub(crate) fn key_contract_activation_gated( + &self, + tenant: &TenantId, + entity_type: &str, + ) -> bool { + self.spec_publication_gated(tenant) + || self + .activating_key_contracts + .read() + .expect("activating key contracts lock poisoned") + .contains(&(tenant.as_str().to_string(), entity_type.to_string())) + } +} diff --git a/crates/temper-server/src/state/tenant_generation/lease.rs b/crates/temper-server/src/state/tenant_generation/lease.rs new file mode 100644 index 000000000..aa7481ef8 --- /dev/null +++ b/crates/temper-server/src/state/tenant_generation/lease.rs @@ -0,0 +1,119 @@ +//! Stable-generation ownership carried by requests and immediate obligations. + +use super::*; + +/// Cloneable owner for the stable tenant generation held by an HTTP request. +/// +/// A bound action whose post-action hook publishes the same tenant may release +/// this lease after its governed action completes, then acquire the publication +/// writer without attempting an impossible read-to-write lock upgrade. +#[derive(Clone)] +pub struct TenantGenerationLease { + tenant: String, + guard: Arc>>>>, + captured_generation: u64, + provenance: TenantGenerationLeaseProvenance, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum TenantGenerationLeaseProvenance { + Request, + PublicationOwned, +} + +impl std::fmt::Debug for TenantGenerationLease { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("TenantGenerationLease") + .field("tenant", &self.tenant) + .field("captured_generation", &self.captured_generation) + .field("held", &self.is_held()) + .field("provenance", &self.provenance) + .finish() + } +} + +impl TenantGenerationLease { + pub(crate) fn new( + tenant: &TenantId, + guard: tokio::sync::OwnedRwLockReadGuard<()>, + captured_generation: u64, + ) -> Self { + Self { + tenant: tenant.as_str().to_string(), + guard: Arc::new(Mutex::new(Some(Arc::new(guard)))), + captured_generation, + provenance: TenantGenerationLeaseProvenance::Request, + } + } + + pub(crate) fn publication_owned(tenant: &TenantId, captured_generation: u64) -> Self { + Self { + tenant: tenant.as_str().to_string(), + guard: Arc::new(Mutex::new(None)), + captured_generation, + provenance: TenantGenerationLeaseProvenance::PublicationOwned, + } + } + + /// Fork an independently releasable proof for immediate detached work. + /// Request forks keep the read side alive so a queued publisher drains the + /// obligation before cutover. Publication-owned work retains its explicit + /// pending-generation provenance and waits for the writer to complete. + pub(crate) fn fork_immediate(&self, tenant: &TenantId) -> Option { + if !self.belongs_to(tenant) { + return None; + } + match self.provenance { + TenantGenerationLeaseProvenance::Request => { + let guard = self + .guard + .lock() + .expect("tenant generation lease lock poisoned") + .clone()?; + Some(Self { + tenant: self.tenant.clone(), + guard: Arc::new(Mutex::new(Some(guard))), + captured_generation: self.captured_generation, + provenance: TenantGenerationLeaseProvenance::Request, + }) + } + TenantGenerationLeaseProvenance::PublicationOwned => { + Some(Self::publication_owned(tenant, self.captured_generation)) + } + } + } + + /// Runtime-generation token observed while this request held its read lease. + pub fn captured_generation(&self) -> u64 { + self.captured_generation + } + + pub(crate) fn is_held_for(&self, tenant: &TenantId) -> bool { + self.belongs_to(tenant) && self.is_held() + } + + pub(crate) fn belongs_to(&self, tenant: &TenantId) -> bool { + self.tenant == tenant.as_str() + } + + pub(crate) fn is_publication_owned_for(&self, tenant: &TenantId) -> bool { + self.belongs_to(tenant) + && self.provenance == TenantGenerationLeaseProvenance::PublicationOwned + } + + fn is_held(&self) -> bool { + self.guard + .lock() + .expect("tenant generation lease lock poisoned") + .is_some() + } + + /// Release the request's stable generation. Repeated release is a no-op. + pub fn release(&self) { + self.guard + .lock() + .expect("tenant generation lease lock poisoned") + .take(); + } +} diff --git a/crates/temper-server/src/storage/dyn_event_store.rs b/crates/temper-server/src/storage/dyn_event_store.rs new file mode 100644 index 000000000..69755d74e --- /dev/null +++ b/crates/temper-server/src/storage/dyn_event_store.rs @@ -0,0 +1,457 @@ +//! Object-safe forwarding adapter for concrete event stores. + +use super::*; + +impl DynEventStore for T +where + T: EventStore, +{ + fn supports_authoritative_key_index(&self) -> bool { + EventStore::supports_authoritative_key_index(self) + } + + fn append<'a>( + &'a self, + persistence_id: &'a str, + expected_sequence: u64, + events: &'a [PersistenceEnvelope], + ) -> EventStoreFuture<'a, Result> { + Box::pin(EventStore::append( + self, + persistence_id, + expected_sequence, + events, + )) + } + + fn append_batch<'a>( + &'a self, + appends: &'a [PersistenceAppend], + ) -> EventStoreFuture<'a, Result, PersistenceError>> { + Box::pin(EventStore::append_batch(self, appends)) + } + + fn batch_idempotency_committed<'a>( + &'a self, + claim: &'a PersistenceBatchIdempotency, + ) -> EventStoreFuture<'a, Result> { + Box::pin(EventStore::batch_idempotency_committed(self, claim)) + } + + fn read_events<'a>( + &'a self, + persistence_id: &'a str, + from_sequence: u64, + ) -> EventStoreFuture<'a, Result, PersistenceError>> { + Box::pin(EventStore::read_events(self, persistence_id, from_sequence)) + } + + fn read_events_page<'a>( + &'a self, + persistence_id: &'a str, + from_sequence: u64, + through_sequence: u64, + limit: usize, + ) -> EventStoreFuture<'a, Result, PersistenceError>> { + Box::pin(EventStore::read_events_page( + self, + persistence_id, + from_sequence, + through_sequence, + limit, + )) + } + + fn journal_boundary<'a>( + &'a self, + persistence_id: &'a str, + ) -> EventStoreFuture<'a, Result> + { + Box::pin(EventStore::journal_boundary(self, persistence_id)) + } + + fn append_with_keys<'a>( + &'a self, + persistence_id: &'a str, + expected_sequence: u64, + events: &'a [PersistenceEnvelope], + key_rows: &'a [temper_runtime::persistence::EntityKeyRow], + ) -> EventStoreFuture<'a, Result> { + Box::pin(EventStore::append_with_keys( + self, + persistence_id, + expected_sequence, + events, + key_rows, + )) + } + + fn append_with_index_rows<'a>( + &'a self, + persistence_id: &'a str, + expected_sequence: u64, + events: &'a [PersistenceEnvelope], + key_rows: &'a [temper_runtime::persistence::EntityKeyRow], + vector_rows: &'a [temper_runtime::persistence::EntityVectorRow], + reconciliation: IndexReconciliation, + ) -> EventStoreFuture<'a, Result> { + Box::pin(EventStore::append_with_index_rows( + self, + persistence_id, + expected_sequence, + events, + key_rows, + vector_rows, + reconciliation, + )) + } + + fn backfill_entity_vectors<'a>( + &'a self, + tenant: &'a str, + entity_type: &'a str, + entity_id: &'a str, + vector_rows: &'a [temper_runtime::persistence::EntityVectorRow], + ) -> EventStoreFuture<'a, Result<(), PersistenceError>> { + Box::pin(EventStore::backfill_entity_vectors( + self, + tenant, + entity_type, + entity_id, + vector_rows, + )) + } + + fn vector_candidates<'a>( + &'a self, + tenant: &'a str, + entity_type: &'a str, + decl_name: &'a str, + model_tag: &'a str, + limit: usize, + ) -> EventStoreFuture< + 'a, + Result, PersistenceError>, + > { + Box::pin(EventStore::vector_candidates( + self, + tenant, + entity_type, + decl_name, + model_tag, + limit, + )) + } + + fn mark_vector_index_backfilled<'a>( + &'a self, + tenant: &'a str, + entity_type: &'a str, + vector_set: &'a str, + ) -> EventStoreFuture<'a, Result<(), PersistenceError>> { + Box::pin(EventStore::mark_vector_index_backfilled( + self, + tenant, + entity_type, + vector_set, + )) + } + + fn vector_index_backfilled_types<'a>( + &'a self, + tenant: &'a str, + ) -> EventStoreFuture<'a, Result, PersistenceError>> { + Box::pin(EventStore::vector_index_backfilled_types(self, tenant)) + } + + fn vectored_entity_ids_for_type<'a>( + &'a self, + tenant: &'a str, + entity_type: &'a str, + ) -> EventStoreFuture<'a, Result, PersistenceError>> { + Box::pin(EventStore::vectored_entity_ids_for_type( + self, + tenant, + entity_type, + )) + } + + fn lookup_by_key<'a>( + &'a self, + tenant: &'a str, + entity_type: &'a str, + key_name: &'a str, + key_hash: &'a str, + ) -> EventStoreFuture<'a, Result, PersistenceError>> { + Box::pin(EventStore::lookup_by_key( + self, + tenant, + entity_type, + key_name, + key_hash, + )) + } + + fn lookup_by_key_with_sequence<'a>( + &'a self, + tenant: &'a str, + entity_type: &'a str, + key_name: &'a str, + key_hash: &'a str, + ) -> EventStoreFuture<'a, Result, PersistenceError>> { + Box::pin(EventStore::lookup_by_key_with_sequence( + self, + tenant, + entity_type, + key_name, + key_hash, + )) + } + + fn backfill_entity_keys<'a>( + &'a self, + tenant: &'a str, + entity_type: &'a str, + entity_id: &'a str, + expected_sequence: u64, + contract_fence: temper_runtime::persistence::KeyIndexBackfillFence<'a>, + key_rows: &'a [temper_runtime::persistence::EntityKeyRow], + ) -> EventStoreFuture<'a, Result<(), PersistenceError>> { + Box::pin(EventStore::backfill_entity_keys( + self, + tenant, + entity_type, + entity_id, + expected_sequence, + contract_fence, + key_rows, + )) + } + + fn mark_key_index_backfilled<'a>( + &'a self, + tenant: &'a str, + entity_type: &'a str, + key_set: &'a str, + ) -> EventStoreFuture<'a, Result<(), PersistenceError>> { + Box::pin(EventStore::mark_key_index_backfilled( + self, + tenant, + entity_type, + key_set, + )) + } + + fn key_index_backfilled_types<'a>( + &'a self, + tenant: &'a str, + ) -> EventStoreFuture<'a, Result, PersistenceError>> { + Box::pin(EventStore::key_index_backfilled_types(self, tenant)) + } + + fn key_index_activated_contracts( + &self, + ) -> EventStoreFuture<'_, Result, PersistenceError>> { + Box::pin(EventStore::key_index_activated_contracts(self)) + } + + fn key_index_reconciliation_revision<'a>( + &'a self, + tenant: &'a str, + entity_type: &'a str, + ) -> EventStoreFuture<'a, Result> { + Box::pin(EventStore::key_index_reconciliation_revision( + self, + tenant, + entity_type, + )) + } + + fn begin_key_index_backfill<'a>( + &'a self, + tenant: &'a str, + entity_type: &'a str, + key_set: &'a str, + ) -> EventStoreFuture<'a, Result> { + Box::pin(EventStore::begin_key_index_backfill( + self, + tenant, + entity_type, + key_set, + )) + } + + fn activate_key_index_contract<'a>( + &'a self, + tenant: &'a str, + entity_type: &'a str, + key_set: &'a str, + purge_existing_rows: bool, + ) -> EventStoreFuture<'a, Result> { + Box::pin(EventStore::activate_key_index_contract( + self, + tenant, + entity_type, + key_set, + purge_existing_rows, + )) + } + + fn activate_key_index_contracts<'a>( + &'a self, + tenant: &'a str, + activations: &'a [temper_runtime::persistence::KeyContractActivation], + ) -> EventStoreFuture<'a, Result, PersistenceError>> + { + Box::pin(EventStore::activate_key_index_contracts( + self, + tenant, + activations, + )) + } + + fn mark_key_index_backfilled_if_revision<'a>( + &'a self, + tenant: &'a str, + entity_type: &'a str, + key_set: &'a str, + expected_revision: u64, + ) -> EventStoreFuture<'a, Result> { + Box::pin(EventStore::mark_key_index_backfilled_if_revision( + self, + tenant, + entity_type, + key_set, + expected_revision, + )) + } + + fn keyed_entity_ids_for_type<'a>( + &'a self, + tenant: &'a str, + entity_type: &'a str, + ) -> EventStoreFuture<'a, Result, PersistenceError>> { + Box::pin(EventStore::keyed_entity_ids_for_type( + self, + tenant, + entity_type, + )) + } + + fn save_snapshot<'a>( + &'a self, + persistence_id: &'a str, + sequence_nr: u64, + snapshot: &'a [u8], + ) -> EventStoreFuture<'a, Result<(), PersistenceError>> { + Box::pin(EventStore::save_snapshot( + self, + persistence_id, + sequence_nr, + snapshot, + )) + } + + fn save_snapshot_if_source<'a>( + &'a self, + persistence_id: &'a str, + sequence_nr: u64, + snapshot: &'a [u8], + source: &'a SnapshotSourceFence, + key_contract: Option<&'a str>, + ) -> EventStoreFuture<'a, Result<(), PersistenceError>> { + Box::pin(EventStore::save_snapshot_if_source( + self, + persistence_id, + sequence_nr, + snapshot, + source, + key_contract, + )) + } + + fn load_snapshot<'a>( + &'a self, + persistence_id: &'a str, + ) -> EventStoreFuture<'a, Result)>, PersistenceError>> { + Box::pin(EventStore::load_snapshot(self, persistence_id)) + } + + fn list_entity_ids<'a>( + &'a self, + tenant: &'a str, + ) -> EventStoreFuture<'a, Result, PersistenceError>> { + Box::pin(EventStore::list_entity_ids(self, tenant)) + } + + fn list_entity_ids_by_type<'a>( + &'a self, + tenant: &'a str, + entity_type: &'a str, + ) -> EventStoreFuture<'a, Result, PersistenceError>> { + Box::pin(EventStore::list_entity_ids_by_type( + self, + tenant, + entity_type, + )) + } + + fn list_entity_ids_for_key_reconciliation<'a>( + &'a self, + tenant: &'a str, + entity_type: &'a str, + ) -> EventStoreFuture<'a, Result, PersistenceError>> { + Box::pin(EventStore::list_entity_ids_for_key_reconciliation( + self, + tenant, + entity_type, + )) + } + + fn key_reconciliation_boundary<'a>( + &'a self, + tenant: &'a str, + entity_type: &'a str, + ) -> EventStoreFuture<'a, Result, PersistenceError>> { + Box::pin(EventStore::key_reconciliation_boundary( + self, + tenant, + entity_type, + )) + } + + fn list_key_reconciliation_page<'a>( + &'a self, + tenant: &'a str, + entity_type: &'a str, + after_entity_id: Option<&'a str>, + through_entity_id: &'a str, + limit: usize, + ) -> EventStoreFuture< + 'a, + Result, PersistenceError>, + > { + Box::pin(EventStore::list_key_reconciliation_page( + self, + tenant, + entity_type, + after_entity_id, + through_entity_id, + limit, + )) + } + + fn list_entity_ids_limited<'a>( + &'a self, + tenant: &'a str, + entity_type: Option<&'a str>, + limit: usize, + ) -> EventStoreFuture<'a, Result, PersistenceError>> { + Box::pin(EventStore::list_entity_ids_limited( + self, + tenant, + entity_type, + limit, + )) + } +} diff --git a/crates/temper-server/src/storage/mod.rs b/crates/temper-server/src/storage/mod.rs index f689b7db0..89fdfbad9 100644 --- a/crates/temper-server/src/storage/mod.rs +++ b/crates/temper-server/src/storage/mod.rs @@ -18,7 +18,9 @@ use std::time::Duration; use sqlx::PgPool; use temper_runtime::persistence::{ - EventStore, PersistenceAppend, PersistenceAppendResult, PersistenceEnvelope, PersistenceError, + EntityKeyLookup, EventStore, IndexReconciliation, JournalBoundary, KeyReconciliationEntity, + PersistenceAppend, PersistenceAppendResult, PersistenceBatchIdempotency, PersistenceEnvelope, + PersistenceError, SnapshotSourceFence, }; use temper_store_postgres::{PostgresEventStore, PostgresPolicyRow, PostgresTrajectoryInsert}; use temper_store_turso::{ @@ -34,6 +36,8 @@ use crate::platform_store::PlatformStore; use crate::platform_store::SimPlatformStore; use crate::state::trajectory::{TrajectoryEntry, TrajectorySource}; +mod dyn_event_store; +mod policy_store; mod published_artifacts; mod query_plane_impls; mod query_plane_read; @@ -53,6 +57,9 @@ pub type EventStoreFuture<'a, T> = Pin + Send + 'a>>; /// Object-safe adapter for the runtime event journal. pub trait DynEventStore: Send + Sync { + /// Whether declared-key rows and coverage watermarks are authoritative here. + fn supports_authoritative_key_index(&self) -> bool; + fn append<'a>( &'a self, persistence_id: &'a str, @@ -65,12 +72,31 @@ pub trait DynEventStore: Send + Sync { appends: &'a [PersistenceAppend], ) -> EventStoreFuture<'a, Result, PersistenceError>>; + fn batch_idempotency_committed<'a>( + &'a self, + claim: &'a PersistenceBatchIdempotency, + ) -> EventStoreFuture<'a, Result>; + fn read_events<'a>( &'a self, persistence_id: &'a str, from_sequence: u64, ) -> EventStoreFuture<'a, Result, PersistenceError>>; + /// Read one storage-bounded event page through an inclusive captured boundary. + fn read_events_page<'a>( + &'a self, + persistence_id: &'a str, + from_sequence: u64, + through_sequence: u64, + limit: usize, + ) -> EventStoreFuture<'a, Result, PersistenceError>>; + + fn journal_boundary<'a>( + &'a self, + persistence_id: &'a str, + ) -> EventStoreFuture<'a, Result>; + fn append_with_keys<'a>( &'a self, persistence_id: &'a str, @@ -86,7 +112,7 @@ pub trait DynEventStore: Send + Sync { events: &'a [PersistenceEnvelope], key_rows: &'a [temper_runtime::persistence::EntityKeyRow], vector_rows: &'a [temper_runtime::persistence::EntityVectorRow], - reconcile_vectors: bool, + reconciliation: IndexReconciliation, ) -> EventStoreFuture<'a, Result>; fn backfill_entity_vectors<'a>( @@ -135,11 +161,22 @@ pub trait DynEventStore: Send + Sync { key_hash: &'a str, ) -> EventStoreFuture<'a, Result, PersistenceError>>; + /// Look up the owner and journal generation committed with a declared key. + fn lookup_by_key_with_sequence<'a>( + &'a self, + tenant: &'a str, + entity_type: &'a str, + key_name: &'a str, + key_hash: &'a str, + ) -> EventStoreFuture<'a, Result, PersistenceError>>; + fn backfill_entity_keys<'a>( &'a self, tenant: &'a str, entity_type: &'a str, entity_id: &'a str, + expected_sequence: u64, + contract_fence: temper_runtime::persistence::KeyIndexBackfillFence<'a>, key_rows: &'a [temper_runtime::persistence::EntityKeyRow], ) -> EventStoreFuture<'a, Result<(), PersistenceError>>; @@ -155,300 +192,117 @@ pub trait DynEventStore: Send + Sync { tenant: &'a str, ) -> EventStoreFuture<'a, Result, PersistenceError>>; - fn keyed_entity_ids_for_type<'a>( + /// Return every durable tenant/type with an activated key contract. + fn key_index_activated_contracts( + &self, + ) -> EventStoreFuture<'_, Result, PersistenceError>>; + + /// Return the revision fencing reconciliation for an entity type's key index. + fn key_index_reconciliation_revision<'a>( &'a self, tenant: &'a str, entity_type: &'a str, - ) -> EventStoreFuture<'a, Result, PersistenceError>>; - - fn save_snapshot<'a>( - &'a self, - persistence_id: &'a str, - sequence_nr: u64, - snapshot: &'a [u8], - ) -> EventStoreFuture<'a, Result<(), PersistenceError>>; - - fn load_snapshot<'a>( - &'a self, - persistence_id: &'a str, - ) -> EventStoreFuture<'a, Result)>, PersistenceError>>; + ) -> EventStoreFuture<'a, Result>; - fn list_entity_ids<'a>( + /// Begin a fenced key-index backfill and return its reconciliation revision. + fn begin_key_index_backfill<'a>( &'a self, tenant: &'a str, - ) -> EventStoreFuture<'a, Result, PersistenceError>>; + entity_type: &'a str, + key_set: &'a str, + ) -> EventStoreFuture<'a, Result>; - fn list_entity_ids_by_type<'a>( + /// Establish the new contract before publishing its spec, atomically + /// purging all prior type rows when the new contract is empty. + fn activate_key_index_contract<'a>( &'a self, tenant: &'a str, entity_type: &'a str, - ) -> EventStoreFuture<'a, Result, PersistenceError>>; + key_set: &'a str, + purge_existing_rows: bool, + ) -> EventStoreFuture<'a, Result>; - fn list_entity_ids_limited<'a>( + /// Atomically activate every changed key contract in one spec publication. + fn activate_key_index_contracts<'a>( &'a self, tenant: &'a str, - entity_type: Option<&'a str>, - limit: usize, - ) -> EventStoreFuture<'a, Result, PersistenceError>>; -} - -impl DynEventStore for T -where - T: EventStore, -{ - fn append<'a>( - &'a self, - persistence_id: &'a str, - expected_sequence: u64, - events: &'a [PersistenceEnvelope], - ) -> EventStoreFuture<'a, Result> { - Box::pin(EventStore::append( - self, - persistence_id, - expected_sequence, - events, - )) - } - - fn append_batch<'a>( - &'a self, - appends: &'a [PersistenceAppend], - ) -> EventStoreFuture<'a, Result, PersistenceError>> { - Box::pin(EventStore::append_batch(self, appends)) - } + activations: &'a [temper_runtime::persistence::KeyContractActivation], + ) -> EventStoreFuture<'a, Result, PersistenceError>>; - fn read_events<'a>( - &'a self, - persistence_id: &'a str, - from_sequence: u64, - ) -> EventStoreFuture<'a, Result, PersistenceError>> { - Box::pin(EventStore::read_events(self, persistence_id, from_sequence)) - } - - fn append_with_keys<'a>( - &'a self, - persistence_id: &'a str, - expected_sequence: u64, - events: &'a [PersistenceEnvelope], - key_rows: &'a [temper_runtime::persistence::EntityKeyRow], - ) -> EventStoreFuture<'a, Result> { - Box::pin(EventStore::append_with_keys( - self, - persistence_id, - expected_sequence, - events, - key_rows, - )) - } - - fn append_with_index_rows<'a>( - &'a self, - persistence_id: &'a str, - expected_sequence: u64, - events: &'a [PersistenceEnvelope], - key_rows: &'a [temper_runtime::persistence::EntityKeyRow], - vector_rows: &'a [temper_runtime::persistence::EntityVectorRow], - reconcile_vectors: bool, - ) -> EventStoreFuture<'a, Result> { - Box::pin(EventStore::append_with_index_rows( - self, - persistence_id, - expected_sequence, - events, - key_rows, - vector_rows, - reconcile_vectors, - )) - } - - fn backfill_entity_vectors<'a>( + /// Mark a key contract complete only if its reconciliation revision is unchanged. + fn mark_key_index_backfilled_if_revision<'a>( &'a self, tenant: &'a str, entity_type: &'a str, - entity_id: &'a str, - vector_rows: &'a [temper_runtime::persistence::EntityVectorRow], - ) -> EventStoreFuture<'a, Result<(), PersistenceError>> { - Box::pin(EventStore::backfill_entity_vectors( - self, - tenant, - entity_type, - entity_id, - vector_rows, - )) - } + key_set: &'a str, + expected_revision: u64, + ) -> EventStoreFuture<'a, Result>; - fn vector_candidates<'a>( + fn keyed_entity_ids_for_type<'a>( &'a self, tenant: &'a str, entity_type: &'a str, - decl_name: &'a str, - model_tag: &'a str, - limit: usize, - ) -> EventStoreFuture< - 'a, - Result, PersistenceError>, - > { - Box::pin(EventStore::vector_candidates( - self, - tenant, - entity_type, - decl_name, - model_tag, - limit, - )) - } + ) -> EventStoreFuture<'a, Result, PersistenceError>>; - fn mark_vector_index_backfilled<'a>( + fn save_snapshot<'a>( &'a self, - tenant: &'a str, - entity_type: &'a str, - vector_set: &'a str, - ) -> EventStoreFuture<'a, Result<(), PersistenceError>> { - Box::pin(EventStore::mark_vector_index_backfilled( - self, - tenant, - entity_type, - vector_set, - )) - } + persistence_id: &'a str, + sequence_nr: u64, + snapshot: &'a [u8], + ) -> EventStoreFuture<'a, Result<(), PersistenceError>>; - fn vector_index_backfilled_types<'a>( + fn save_snapshot_if_source<'a>( &'a self, - tenant: &'a str, - ) -> EventStoreFuture<'a, Result, PersistenceError>> { - Box::pin(EventStore::vector_index_backfilled_types(self, tenant)) - } + persistence_id: &'a str, + sequence_nr: u64, + snapshot: &'a [u8], + source: &'a SnapshotSourceFence, + key_contract: Option<&'a str>, + ) -> EventStoreFuture<'a, Result<(), PersistenceError>>; - fn vectored_entity_ids_for_type<'a>( + fn load_snapshot<'a>( &'a self, - tenant: &'a str, - entity_type: &'a str, - ) -> EventStoreFuture<'a, Result, PersistenceError>> { - Box::pin(EventStore::vectored_entity_ids_for_type( - self, - tenant, - entity_type, - )) - } + persistence_id: &'a str, + ) -> EventStoreFuture<'a, Result)>, PersistenceError>>; - fn lookup_by_key<'a>( + fn list_entity_ids<'a>( &'a self, tenant: &'a str, - entity_type: &'a str, - key_name: &'a str, - key_hash: &'a str, - ) -> EventStoreFuture<'a, Result, PersistenceError>> { - Box::pin(EventStore::lookup_by_key( - self, - tenant, - entity_type, - key_name, - key_hash, - )) - } + ) -> EventStoreFuture<'a, Result, PersistenceError>>; - fn backfill_entity_keys<'a>( + fn list_entity_ids_by_type<'a>( &'a self, tenant: &'a str, entity_type: &'a str, - entity_id: &'a str, - key_rows: &'a [temper_runtime::persistence::EntityKeyRow], - ) -> EventStoreFuture<'a, Result<(), PersistenceError>> { - Box::pin(EventStore::backfill_entity_keys( - self, - tenant, - entity_type, - entity_id, - key_rows, - )) - } + ) -> EventStoreFuture<'a, Result, PersistenceError>>; - fn mark_key_index_backfilled<'a>( + fn list_entity_ids_for_key_reconciliation<'a>( &'a self, tenant: &'a str, entity_type: &'a str, - key_set: &'a str, - ) -> EventStoreFuture<'a, Result<(), PersistenceError>> { - Box::pin(EventStore::mark_key_index_backfilled( - self, - tenant, - entity_type, - key_set, - )) - } - - fn key_index_backfilled_types<'a>( - &'a self, - tenant: &'a str, - ) -> EventStoreFuture<'a, Result, PersistenceError>> { - Box::pin(EventStore::key_index_backfilled_types(self, tenant)) - } + ) -> EventStoreFuture<'a, Result, PersistenceError>>; - fn keyed_entity_ids_for_type<'a>( + fn key_reconciliation_boundary<'a>( &'a self, tenant: &'a str, entity_type: &'a str, - ) -> EventStoreFuture<'a, Result, PersistenceError>> { - Box::pin(EventStore::keyed_entity_ids_for_type( - self, - tenant, - entity_type, - )) - } - - fn save_snapshot<'a>( - &'a self, - persistence_id: &'a str, - sequence_nr: u64, - snapshot: &'a [u8], - ) -> EventStoreFuture<'a, Result<(), PersistenceError>> { - Box::pin(EventStore::save_snapshot( - self, - persistence_id, - sequence_nr, - snapshot, - )) - } - - fn load_snapshot<'a>( - &'a self, - persistence_id: &'a str, - ) -> EventStoreFuture<'a, Result)>, PersistenceError>> { - Box::pin(EventStore::load_snapshot(self, persistence_id)) - } + ) -> EventStoreFuture<'a, Result, PersistenceError>>; - fn list_entity_ids<'a>( - &'a self, - tenant: &'a str, - ) -> EventStoreFuture<'a, Result, PersistenceError>> { - Box::pin(EventStore::list_entity_ids(self, tenant)) - } - - fn list_entity_ids_by_type<'a>( + fn list_key_reconciliation_page<'a>( &'a self, tenant: &'a str, entity_type: &'a str, - ) -> EventStoreFuture<'a, Result, PersistenceError>> { - Box::pin(EventStore::list_entity_ids_by_type( - self, - tenant, - entity_type, - )) - } + after_entity_id: Option<&'a str>, + through_entity_id: &'a str, + limit: usize, + ) -> EventStoreFuture<'a, Result, PersistenceError>>; fn list_entity_ids_limited<'a>( &'a self, tenant: &'a str, entity_type: Option<&'a str>, limit: usize, - ) -> EventStoreFuture<'a, Result, PersistenceError>> { - Box::pin(EventStore::list_entity_ids_limited( - self, - tenant, - entity_type, - limit, - )) - } + ) -> EventStoreFuture<'a, Result, PersistenceError>>; } /// Cloneable boxed event store handle. @@ -474,6 +328,11 @@ impl BoxedEventStore { self.0.clone() } + /// Whether this backend can serve as the exact declared-key ownership oracle. + pub fn supports_authoritative_key_index(&self) -> bool { + self.0.supports_authoritative_key_index() + } + pub async fn append( &self, persistence_id: &str, @@ -492,6 +351,13 @@ impl BoxedEventStore { self.0.append_batch(appends).await } + pub async fn batch_idempotency_committed( + &self, + claim: &PersistenceBatchIdempotency, + ) -> Result { + self.0.batch_idempotency_committed(claim).await + } + pub async fn read_events( &self, persistence_id: &str, @@ -500,6 +366,27 @@ impl BoxedEventStore { self.0.read_events(persistence_id, from_sequence).await } + /// Read one storage-bounded page within a caller-captured journal boundary. + pub async fn read_events_page( + &self, + persistence_id: &str, + from_sequence: u64, + through_sequence: u64, + limit: usize, + ) -> Result, PersistenceError> { + self.0 + .read_events_page(persistence_id, from_sequence, through_sequence, limit) + .await + } + + /// Return the exact high-water and terminal boundary for one persistence stream. + pub async fn journal_boundary( + &self, + persistence_id: &str, + ) -> Result { + self.0.journal_boundary(persistence_id).await + } + pub async fn append_with_keys( &self, persistence_id: &str, @@ -519,7 +406,7 @@ impl BoxedEventStore { events: &[PersistenceEnvelope], key_rows: &[temper_runtime::persistence::EntityKeyRow], vector_rows: &[temper_runtime::persistence::EntityVectorRow], - reconcile_vectors: bool, + reconciliation: IndexReconciliation, ) -> Result { self.0 .append_with_index_rows( @@ -528,7 +415,7 @@ impl BoxedEventStore { events, key_rows, vector_rows, - reconcile_vectors, + reconciliation, ) .await } @@ -598,15 +485,37 @@ impl BoxedEventStore { .await } + /// Look up the owner and journal generation committed with a declared key. + pub async fn lookup_by_key_with_sequence( + &self, + tenant: &str, + entity_type: &str, + key_name: &str, + key_hash: &str, + ) -> Result, PersistenceError> { + self.0 + .lookup_by_key_with_sequence(tenant, entity_type, key_name, key_hash) + .await + } + pub async fn backfill_entity_keys( &self, tenant: &str, entity_type: &str, entity_id: &str, + expected_sequence: u64, + contract_fence: temper_runtime::persistence::KeyIndexBackfillFence<'_>, key_rows: &[temper_runtime::persistence::EntityKeyRow], ) -> Result<(), PersistenceError> { self.0 - .backfill_entity_keys(tenant, entity_type, entity_id, key_rows) + .backfill_entity_keys( + tenant, + entity_type, + entity_id, + expected_sequence, + contract_fence, + key_rows, + ) .await } @@ -628,6 +537,73 @@ impl BoxedEventStore { self.0.key_index_backfilled_types(tenant).await } + /// Return every durable tenant/type with an activated key contract. + pub async fn key_index_activated_contracts( + &self, + ) -> Result, PersistenceError> { + self.0.key_index_activated_contracts().await + } + + /// Return the revision fencing reconciliation for an entity type's key index. + pub async fn key_index_reconciliation_revision( + &self, + tenant: &str, + entity_type: &str, + ) -> Result { + self.0 + .key_index_reconciliation_revision(tenant, entity_type) + .await + } + + /// Begin a fenced key-index backfill and return its reconciliation revision. + pub async fn begin_key_index_backfill( + &self, + tenant: &str, + entity_type: &str, + key_set: &str, + ) -> Result { + self.0 + .begin_key_index_backfill(tenant, entity_type, key_set) + .await + } + + /// Establish a key contract before the corresponding spec becomes live. + pub async fn activate_key_index_contract( + &self, + tenant: &str, + entity_type: &str, + key_set: &str, + purge_existing_rows: bool, + ) -> Result { + self.0 + .activate_key_index_contract(tenant, entity_type, key_set, purge_existing_rows) + .await + } + + /// Atomically activate every changed key contract in one spec publication. + pub async fn activate_key_index_contracts( + &self, + tenant: &str, + activations: &[temper_runtime::persistence::KeyContractActivation], + ) -> Result, PersistenceError> { + self.0 + .activate_key_index_contracts(tenant, activations) + .await + } + + /// Mark a key contract complete only if its reconciliation revision is unchanged. + pub async fn mark_key_index_backfilled_if_revision( + &self, + tenant: &str, + entity_type: &str, + key_set: &str, + expected_revision: u64, + ) -> Result { + self.0 + .mark_key_index_backfilled_if_revision(tenant, entity_type, key_set, expected_revision) + .await + } + pub async fn keyed_entity_ids_for_type( &self, tenant: &str, @@ -647,6 +623,19 @@ impl BoxedEventStore { .await } + pub async fn save_snapshot_if_source( + &self, + persistence_id: &str, + sequence_nr: u64, + snapshot: &[u8], + source: &SnapshotSourceFence, + key_contract: Option<&str>, + ) -> Result<(), PersistenceError> { + self.0 + .save_snapshot_if_source(persistence_id, sequence_nr, snapshot, source, key_contract) + .await + } + pub async fn load_snapshot( &self, persistence_id: &str, @@ -669,6 +658,49 @@ impl BoxedEventStore { self.0.list_entity_ids_by_type(tenant, entity_type).await } + /// Enumerate every journal stream or key-index owner that exact declared-key + /// repair must reconcile, including deleted and key-only phantom owners. + pub async fn list_entity_ids_for_key_reconciliation( + &self, + tenant: &str, + entity_type: &str, + ) -> Result, PersistenceError> { + self.0 + .list_entity_ids_for_key_reconciliation(tenant, entity_type) + .await + } + + /// Read one bounded, deterministic repair page with durable liveness. + pub async fn list_key_reconciliation_page( + &self, + tenant: &str, + entity_type: &str, + after_entity_id: Option<&str>, + through_entity_id: &str, + limit: usize, + ) -> Result, PersistenceError> { + self.0 + .list_key_reconciliation_page( + tenant, + entity_type, + after_entity_id, + through_entity_id, + limit, + ) + .await + } + + /// Capture the inclusive terminal candidate for a bounded repair scan. + pub async fn key_reconciliation_boundary( + &self, + tenant: &str, + entity_type: &str, + ) -> Result, PersistenceError> { + self.0 + .key_reconciliation_boundary(tenant, entity_type) + .await + } + pub async fn list_entity_ids_limited( &self, tenant: &str, @@ -715,6 +747,19 @@ pub struct PolicyStoreRow { pub enabled: bool, } +/// One row in a complete tenant Cedar policy generation. +#[derive(Clone, Debug)] +pub struct PolicyGenerationWrite { + /// Stable identifier within the tenant generation. + pub policy_id: String, + /// Raw Cedar policy source. + pub cedar_text: String, + /// Whether the row participates in the live generation. + pub enabled: bool, + /// Identity responsible for the latest row mutation. + pub created_by: String, +} + impl From for PolicyStoreRow { fn from(row: TursoPolicyRow) -> Self { Self { @@ -762,6 +807,14 @@ pub struct DataOnlyCreateRecord<'a> { pub state: &'a serde_json::Value, /// First event envelope to append at sequence number 1. pub event: &'a PersistenceEnvelope, + /// Complete declared-key row set derived from the initial durable state. + pub key_rows: &'a [temper_runtime::persistence::EntityKeyRow], + /// Whether this write authoritatively reconciles exact ownership. This is + /// true for spec-governed writes even when the current key set is empty. + pub reconcile_keys: bool, + /// Versioned signature of the current declared-key contract, including the + /// empty signature for a no-key type. + pub key_set_signature: &'a str, } /// Optional native storage capability for brand-new data-only creates. @@ -792,6 +845,22 @@ pub trait BackendNamedStore: Send + Sync { /// Granular Cedar policy persistence capability. #[async_trait::async_trait] pub trait PolicyStore: Send + Sync { + /// Atomically replace the canonical granular rows and compatibility text. + /// + /// An empty `entries` slice is an explicit empty generation: implementations + /// must still persist `compatibility_text` so restart cannot promote an older + /// aggregate after the final granular row is removed. + async fn replace_policy_generation( + &self, + tenant: &str, + entries: &[PolicyGenerationWrite], + compatibility_text: &str, + ) -> Result<(), String>; + + /// Load the legacy aggregate projection for one tenant, including an + /// explicitly persisted empty string. + async fn load_policy_compatibility_text(&self, tenant: &str) -> Result, String>; + async fn save_policy( &self, tenant: &str, @@ -819,6 +888,21 @@ pub trait PolicyStore: Send + Sync { created_by: &str, ) -> Result; + /// Atomically replace both mutable fields of one policy row. + /// + /// This is the durable half of a policy-generation cutover. Callers must + /// not emulate it with separate text and enabled writes because a fault + /// between those statements would make the persisted generation + /// unrecoverable as a single unit. + async fn replace_policy( + &self, + tenant: &str, + policy_id: &str, + cedar_text: &str, + enabled: bool, + created_by: &str, + ) -> Result; + async fn delete_policy(&self, tenant: &str, policy_id: &str) -> Result<(), String>; } @@ -1514,218 +1598,6 @@ impl TursoStoreProvider for TenantRoutedTursoStoreProvider { } } -#[async_trait::async_trait] -impl PolicyStore for PostgresEventStore { - async fn save_policy( - &self, - tenant: &str, - policy_id: &str, - cedar_text: &str, - created_by: &str, - ) -> Result { - self.save_policy(tenant, policy_id, cedar_text, created_by) - .await - .map_err(|e| e.to_string()) - } - - async fn load_policies_for_tenant(&self, tenant: &str) -> Result, String> { - self.load_policies_for_tenant(tenant) - .await - .map(|rows| rows.into_iter().map(PolicyStoreRow::from).collect()) - .map_err(|e| e.to_string()) - } - - async fn load_all_policies(&self) -> Result, String> { - self.load_all_policies() - .await - .map(|rows| rows.into_iter().map(PolicyStoreRow::from).collect()) - .map_err(|e| e.to_string()) - } - - async fn toggle_policy_enabled( - &self, - tenant: &str, - policy_id: &str, - enabled: bool, - ) -> Result { - self.toggle_policy_enabled(tenant, policy_id, enabled) - .await - .map_err(|e| e.to_string()) - } - - async fn update_policy_text( - &self, - tenant: &str, - policy_id: &str, - cedar_text: &str, - created_by: &str, - ) -> Result { - self.update_policy_text(tenant, policy_id, cedar_text, created_by) - .await - .map_err(|e| e.to_string()) - } - - async fn delete_policy(&self, tenant: &str, policy_id: &str) -> Result<(), String> { - self.delete_policy(tenant, policy_id) - .await - .map_err(|e| e.to_string()) - } -} - -#[async_trait::async_trait] -impl PolicyStore for TursoEventStore { - async fn save_policy( - &self, - tenant: &str, - policy_id: &str, - cedar_text: &str, - created_by: &str, - ) -> Result { - self.save_policy(tenant, policy_id, cedar_text, created_by) - .await - .map_err(|e| e.to_string()) - } - - async fn load_policies_for_tenant(&self, tenant: &str) -> Result, String> { - self.load_policies_for_tenant(tenant) - .await - .map(|rows| rows.into_iter().map(PolicyStoreRow::from).collect()) - .map_err(|e| e.to_string()) - } - - async fn load_all_policies(&self) -> Result, String> { - self.load_all_policies() - .await - .map(|rows| rows.into_iter().map(PolicyStoreRow::from).collect()) - .map_err(|e| e.to_string()) - } - - async fn toggle_policy_enabled( - &self, - tenant: &str, - policy_id: &str, - enabled: bool, - ) -> Result { - self.toggle_policy_enabled(tenant, policy_id, enabled) - .await - .map_err(|e| e.to_string()) - } - - async fn update_policy_text( - &self, - tenant: &str, - policy_id: &str, - cedar_text: &str, - created_by: &str, - ) -> Result { - self.update_policy_text(tenant, policy_id, cedar_text, created_by) - .await - .map_err(|e| e.to_string()) - } - - async fn delete_policy(&self, tenant: &str, policy_id: &str) -> Result<(), String> { - self.delete_policy(tenant, policy_id) - .await - .map_err(|e| e.to_string()) - } -} - -#[async_trait::async_trait] -impl PolicyStore for TenantStoreRouter { - async fn save_policy( - &self, - tenant: &str, - policy_id: &str, - cedar_text: &str, - created_by: &str, - ) -> Result { - let store = self - .store_for_tenant(tenant) - .await - .map_err(|e| e.to_string())?; - store - .save_policy(tenant, policy_id, cedar_text, created_by) - .await - .map_err(|e| e.to_string()) - } - - async fn load_policies_for_tenant(&self, tenant: &str) -> Result, String> { - let store = self - .store_for_tenant(tenant) - .await - .map_err(|e| e.to_string())?; - store - .load_policies_for_tenant(tenant) - .await - .map(|rows| rows.into_iter().map(PolicyStoreRow::from).collect()) - .map_err(|e| e.to_string()) - } - - async fn load_all_policies(&self) -> Result, String> { - let mut rows: Vec = self - .platform_store() - .load_all_policies() - .await - .map(|rows| rows.into_iter().map(PolicyStoreRow::from).collect()) - .map_err(|e| e.to_string())?; - for tenant_id in self.connected_tenants().await { - if let Ok(store) = self.store_for_tenant(&tenant_id).await { - let mut tenant_rows: Vec = store - .load_all_policies() - .await - .map(|rows| rows.into_iter().map(PolicyStoreRow::from).collect()) - .map_err(|e| e.to_string())?; - rows.append(&mut tenant_rows); - } - } - Ok(rows) - } - - async fn toggle_policy_enabled( - &self, - tenant: &str, - policy_id: &str, - enabled: bool, - ) -> Result { - let store = self - .store_for_tenant(tenant) - .await - .map_err(|e| e.to_string())?; - store - .toggle_policy_enabled(tenant, policy_id, enabled) - .await - .map_err(|e| e.to_string()) - } - - async fn update_policy_text( - &self, - tenant: &str, - policy_id: &str, - cedar_text: &str, - created_by: &str, - ) -> Result { - let store = self - .store_for_tenant(tenant) - .await - .map_err(|e| e.to_string())?; - store - .update_policy_text(tenant, policy_id, cedar_text, created_by) - .await - .map_err(|e| e.to_string()) - } - - async fn delete_policy(&self, tenant: &str, policy_id: &str) -> Result<(), String> { - let store = self - .store_for_tenant(tenant) - .await - .map_err(|e| e.to_string())?; - store - .delete_policy(tenant, policy_id) - .await - .map_err(|e| e.to_string()) - } -} - impl BackendNamedStore for PostgresEventStore { fn backend_name(&self) -> &'static str { "postgres" @@ -2664,7 +2536,7 @@ impl DataOnlyCreateStore for PostgresEventStore { &self, record: DataOnlyCreateRecord<'_>, ) -> Result { - self.create_data_only_entity_native_with_state( + self.create_data_only_entity_native_with_state_and_keys( record.tenant, record.entity_type, record.entity_id, @@ -2672,6 +2544,9 @@ impl DataOnlyCreateStore for PostgresEventStore { record.fields, record.state, record.event, + record.key_rows, + record.reconcile_keys, + Some(record.key_set_signature), ) .await } diff --git a/crates/temper-server/src/storage/policy_store.rs b/crates/temper-server/src/storage/policy_store.rs new file mode 100644 index 000000000..2f8105d1f --- /dev/null +++ b/crates/temper-server/src/storage/policy_store.rs @@ -0,0 +1,351 @@ +//! Policy-store adapters for concrete storage backends. + +use super::{PolicyGenerationWrite, PolicyStore, PolicyStoreRow}; +use temper_store_postgres::{PostgresEventStore, PostgresPolicyGenerationWrite}; +use temper_store_turso::{ + PolicyGenerationWrite as TursoPolicyGenerationWrite, TenantStoreRouter, TursoEventStore, +}; + +#[async_trait::async_trait] +impl PolicyStore for PostgresEventStore { + async fn replace_policy_generation( + &self, + tenant: &str, + entries: &[PolicyGenerationWrite], + compatibility_text: &str, + ) -> Result<(), String> { + let entries = entries + .iter() + .map(|entry| PostgresPolicyGenerationWrite { + policy_id: &entry.policy_id, + cedar_text: &entry.cedar_text, + enabled: entry.enabled, + created_by: &entry.created_by, + }) + .collect::>(); + self.replace_policy_generation(tenant, &entries, compatibility_text) + .await + .map_err(|error| error.to_string()) + } + + async fn load_policy_compatibility_text(&self, tenant: &str) -> Result, String> { + self.load_tenant_policy(tenant) + .await + .map_err(|error| error.to_string()) + } + + async fn save_policy( + &self, + tenant: &str, + policy_id: &str, + cedar_text: &str, + created_by: &str, + ) -> Result { + self.save_policy(tenant, policy_id, cedar_text, created_by) + .await + .map_err(|error| error.to_string()) + } + + async fn load_policies_for_tenant(&self, tenant: &str) -> Result, String> { + self.load_policies_for_tenant(tenant) + .await + .map(|rows| rows.into_iter().map(PolicyStoreRow::from).collect()) + .map_err(|error| error.to_string()) + } + + async fn load_all_policies(&self) -> Result, String> { + self.load_all_policies() + .await + .map(|rows| rows.into_iter().map(PolicyStoreRow::from).collect()) + .map_err(|error| error.to_string()) + } + + async fn toggle_policy_enabled( + &self, + tenant: &str, + policy_id: &str, + enabled: bool, + ) -> Result { + self.toggle_policy_enabled(tenant, policy_id, enabled) + .await + .map_err(|error| error.to_string()) + } + + async fn update_policy_text( + &self, + tenant: &str, + policy_id: &str, + cedar_text: &str, + created_by: &str, + ) -> Result { + self.update_policy_text(tenant, policy_id, cedar_text, created_by) + .await + .map_err(|error| error.to_string()) + } + + async fn replace_policy( + &self, + tenant: &str, + policy_id: &str, + cedar_text: &str, + enabled: bool, + created_by: &str, + ) -> Result { + self.replace_policy(tenant, policy_id, cedar_text, enabled, created_by) + .await + .map_err(|error| error.to_string()) + } + + async fn delete_policy(&self, tenant: &str, policy_id: &str) -> Result<(), String> { + self.delete_policy(tenant, policy_id) + .await + .map_err(|error| error.to_string()) + } +} + +#[async_trait::async_trait] +impl PolicyStore for TursoEventStore { + async fn replace_policy_generation( + &self, + tenant: &str, + entries: &[PolicyGenerationWrite], + compatibility_text: &str, + ) -> Result<(), String> { + let entries = entries + .iter() + .map(|entry| TursoPolicyGenerationWrite { + policy_id: &entry.policy_id, + cedar_text: &entry.cedar_text, + enabled: entry.enabled, + created_by: &entry.created_by, + }) + .collect::>(); + self.replace_policy_generation(tenant, &entries, compatibility_text) + .await + .map_err(|error| error.to_string()) + } + + async fn load_policy_compatibility_text(&self, tenant: &str) -> Result, String> { + self.load_tenant_policy(tenant) + .await + .map_err(|error| error.to_string()) + } + + async fn save_policy( + &self, + tenant: &str, + policy_id: &str, + cedar_text: &str, + created_by: &str, + ) -> Result { + self.save_policy(tenant, policy_id, cedar_text, created_by) + .await + .map_err(|error| error.to_string()) + } + + async fn load_policies_for_tenant(&self, tenant: &str) -> Result, String> { + self.load_policies_for_tenant(tenant) + .await + .map(|rows| rows.into_iter().map(PolicyStoreRow::from).collect()) + .map_err(|error| error.to_string()) + } + + async fn load_all_policies(&self) -> Result, String> { + self.load_all_policies() + .await + .map(|rows| rows.into_iter().map(PolicyStoreRow::from).collect()) + .map_err(|error| error.to_string()) + } + + async fn toggle_policy_enabled( + &self, + tenant: &str, + policy_id: &str, + enabled: bool, + ) -> Result { + self.toggle_policy_enabled(tenant, policy_id, enabled) + .await + .map_err(|error| error.to_string()) + } + + async fn update_policy_text( + &self, + tenant: &str, + policy_id: &str, + cedar_text: &str, + created_by: &str, + ) -> Result { + self.update_policy_text(tenant, policy_id, cedar_text, created_by) + .await + .map_err(|error| error.to_string()) + } + + async fn replace_policy( + &self, + tenant: &str, + policy_id: &str, + cedar_text: &str, + enabled: bool, + created_by: &str, + ) -> Result { + self.replace_policy(tenant, policy_id, cedar_text, enabled, created_by) + .await + .map_err(|error| error.to_string()) + } + + async fn delete_policy(&self, tenant: &str, policy_id: &str) -> Result<(), String> { + self.delete_policy(tenant, policy_id) + .await + .map_err(|error| error.to_string()) + } +} + +#[async_trait::async_trait] +impl PolicyStore for TenantStoreRouter { + async fn replace_policy_generation( + &self, + tenant: &str, + entries: &[PolicyGenerationWrite], + compatibility_text: &str, + ) -> Result<(), String> { + let store = self + .store_for_tenant(tenant) + .await + .map_err(|error| error.to_string())?; + let entries = entries + .iter() + .map(|entry| TursoPolicyGenerationWrite { + policy_id: &entry.policy_id, + cedar_text: &entry.cedar_text, + enabled: entry.enabled, + created_by: &entry.created_by, + }) + .collect::>(); + store + .replace_policy_generation(tenant, &entries, compatibility_text) + .await + .map_err(|error| error.to_string()) + } + + async fn load_policy_compatibility_text(&self, tenant: &str) -> Result, String> { + let store = self + .store_for_tenant(tenant) + .await + .map_err(|error| error.to_string())?; + store + .load_tenant_policy(tenant) + .await + .map_err(|error| error.to_string()) + } + + async fn save_policy( + &self, + tenant: &str, + policy_id: &str, + cedar_text: &str, + created_by: &str, + ) -> Result { + let store = self + .store_for_tenant(tenant) + .await + .map_err(|error| error.to_string())?; + store + .save_policy(tenant, policy_id, cedar_text, created_by) + .await + .map_err(|error| error.to_string()) + } + + async fn load_policies_for_tenant(&self, tenant: &str) -> Result, String> { + let store = self + .store_for_tenant(tenant) + .await + .map_err(|error| error.to_string())?; + store + .load_policies_for_tenant(tenant) + .await + .map(|rows| rows.into_iter().map(PolicyStoreRow::from).collect()) + .map_err(|error| error.to_string()) + } + + async fn load_all_policies(&self) -> Result, String> { + let mut rows: Vec = self + .platform_store() + .load_all_policies() + .await + .map(|rows| rows.into_iter().map(PolicyStoreRow::from).collect()) + .map_err(|error| error.to_string())?; + for tenant_id in self.connected_tenants().await { + if let Ok(store) = self.store_for_tenant(&tenant_id).await { + let mut tenant_rows: Vec = store + .load_all_policies() + .await + .map(|rows| rows.into_iter().map(PolicyStoreRow::from).collect()) + .map_err(|error| error.to_string())?; + rows.append(&mut tenant_rows); + } + } + Ok(rows) + } + + async fn toggle_policy_enabled( + &self, + tenant: &str, + policy_id: &str, + enabled: bool, + ) -> Result { + let store = self + .store_for_tenant(tenant) + .await + .map_err(|error| error.to_string())?; + store + .toggle_policy_enabled(tenant, policy_id, enabled) + .await + .map_err(|error| error.to_string()) + } + + async fn update_policy_text( + &self, + tenant: &str, + policy_id: &str, + cedar_text: &str, + created_by: &str, + ) -> Result { + let store = self + .store_for_tenant(tenant) + .await + .map_err(|error| error.to_string())?; + store + .update_policy_text(tenant, policy_id, cedar_text, created_by) + .await + .map_err(|error| error.to_string()) + } + + async fn replace_policy( + &self, + tenant: &str, + policy_id: &str, + cedar_text: &str, + enabled: bool, + created_by: &str, + ) -> Result { + let store = self + .store_for_tenant(tenant) + .await + .map_err(|error| error.to_string())?; + store + .replace_policy(tenant, policy_id, cedar_text, enabled, created_by) + .await + .map_err(|error| error.to_string()) + } + + async fn delete_policy(&self, tenant: &str, policy_id: &str) -> Result<(), String> { + let store = self + .store_for_tenant(tenant) + .await + .map_err(|error| error.to_string())?; + store + .delete_policy(tenant, policy_id) + .await + .map_err(|error| error.to_string()) + } +} diff --git a/crates/temper-server/src/storage/query_plane.rs b/crates/temper-server/src/storage/query_plane.rs index 220181b82..1d5843f33 100644 --- a/crates/temper-server/src/storage/query_plane.rs +++ b/crates/temper-server/src/storage/query_plane.rs @@ -2,7 +2,7 @@ use std::collections::BTreeMap; -use temper_runtime::persistence::PersistenceError; +use temper_runtime::persistence::{PersistenceError, ProjectionSourceFence}; /// Backend-neutral projection row returned by [`QueryPlaneStore`]. #[derive(Clone, Debug, PartialEq)] @@ -82,6 +82,29 @@ pub trait QueryPlaneStore: Send + Sync { sequence_nr: u64, ) -> Result<(), PersistenceError>; + /// Upsert a projection only while the exact journal and snapshot generation + /// used to derive it is still current. The source comparison and mutation + /// must share one backend transaction. `false` means no mutation occurred. + #[expect( + clippy::too_many_arguments, + reason = "source-fenced projection boundary" + )] + async fn upsert_projection_if_source( + &self, + _tenant: &str, + _entity_type: &str, + _entity_id: &str, + _status: &str, + _fields: &serde_json::Value, + _state: &serde_json::Value, + _sequence_nr: u64, + _source: ProjectionSourceFence<'_>, + ) -> Result { + Err(PersistenceError::Storage( + "query plane does not support source-fenced projection repair".to_string(), + )) + } + async fn upsert_projections( &self, tenant: &str, @@ -109,6 +132,55 @@ pub trait QueryPlaneStore: Send + Sync { entity_id: &str, ) -> Result<(), PersistenceError>; + /// Remove a projection only while the exact journal and snapshot generation + /// used to classify the entity as deleted is still current. + async fn remove_projection_if_source( + &self, + _tenant: &str, + _entity_type: &str, + _entity_id: &str, + _source: ProjectionSourceFence<'_>, + ) -> Result { + Err(PersistenceError::Storage( + "query plane does not support source-fenced projection repair".to_string(), + )) + } + + /// Clear a durable dirty marker only while the exact journal and snapshot + /// source is current, without changing a compatibility catalog row. + async fn clear_projection_dirty_if_source( + &self, + _tenant: &str, + _entity_type: &str, + _entity_id: &str, + _source: ProjectionSourceFence<'_>, + ) -> Result { + Err(PersistenceError::Storage( + "query plane does not support source-fenced dirty acknowledgement".to_string(), + )) + } + + /// Remove exactly the attempted projection row after its durable source was + /// observed to change. A different or newer catalog row is never removed. + #[expect( + clippy::too_many_arguments, + reason = "exact projection cleanup boundary" + )] + async fn remove_projection_if_exact( + &self, + _tenant: &str, + _entity_type: &str, + _entity_id: &str, + _status: &str, + _fields: &serde_json::Value, + _state: &serde_json::Value, + _sequence_nr: u64, + ) -> Result { + Err(PersistenceError::Storage( + "query plane does not support exact projection cleanup".to_string(), + )) + } + async fn query_field_index( &self, tenant: &str, @@ -178,4 +250,16 @@ pub trait QueryPlaneStore: Send + Sync { async fn projected_entity_counts_by_tenant( &self, ) -> Result>, PersistenceError>; + + /// Return at most `limit` entities whose durable source changed after their + /// last source-fenced projection reconciliation. `None` means the backend + /// does not expose durable dirty-projection tracking. + async fn dirty_projection_entity_ids( + &self, + _tenant: &str, + _entity_type: &str, + _limit: usize, + ) -> Result>, PersistenceError> { + Ok(None) + } } diff --git a/crates/temper-server/src/storage/query_plane_impls.rs b/crates/temper-server/src/storage/query_plane_impls.rs index 80dfc739d..1ad9fa514 100644 --- a/crates/temper-server/src/storage/query_plane_impls.rs +++ b/crates/temper-server/src/storage/query_plane_impls.rs @@ -1,14 +1,14 @@ -use temper_runtime::persistence::PersistenceError; +use temper_runtime::persistence::{PersistenceError, ProjectionSourceFence}; use temper_store_postgres::PostgresEventStore; -use temper_store_turso::{ - QueryProjectionUpsert as TursoQueryProjectionUpsert, TenantStoreRouter, TursoEventStore, -}; +use temper_store_turso::{QueryProjectionUpsert as TursoQueryProjectionUpsert, TursoEventStore}; use super::{ EntityCatalogRow, QueryFieldIndexOrder, QueryFieldIndexOrderDirection, QueryFieldIndexPage, QueryPlaneStore, QueryProjectionFieldsRow, QueryProjectionUpsert, }; +mod router; + fn storage_order_by(order_by: &[QueryFieldIndexOrder]) -> Vec<(String, bool)> { order_by .iter() @@ -45,6 +45,30 @@ impl QueryPlaneStore for PostgresEventStore { .await } + async fn upsert_projection_if_source( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + status: &str, + fields: &serde_json::Value, + state: &serde_json::Value, + sequence_nr: u64, + source: ProjectionSourceFence<'_>, + ) -> Result { + self.upsert_query_projection_with_state_if_source( + tenant, + entity_type, + entity_id, + status, + fields, + state, + sequence_nr, + source, + ) + .await + } + async fn remove_projection( &self, tenant: &str, @@ -55,6 +79,50 @@ impl QueryPlaneStore for PostgresEventStore { .await } + async fn remove_projection_if_source( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + source: ProjectionSourceFence<'_>, + ) -> Result { + self.remove_query_projection_if_source(tenant, entity_type, entity_id, source) + .await + } + + async fn clear_projection_dirty_if_source( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + source: ProjectionSourceFence<'_>, + ) -> Result { + self.clear_query_projection_dirty_if_source(tenant, entity_type, entity_id, source) + .await + } + + async fn remove_projection_if_exact( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + status: &str, + fields: &serde_json::Value, + state: &serde_json::Value, + sequence_nr: u64, + ) -> Result { + self.remove_query_projection_if_exact( + tenant, + entity_type, + entity_id, + status, + fields, + state, + sequence_nr, + ) + .await + } + async fn query_field_index( &self, tenant: &str, @@ -173,6 +241,17 @@ impl QueryPlaneStore for PostgresEventStore { .await .map(Some) } + + async fn dirty_projection_entity_ids( + &self, + tenant: &str, + entity_type: &str, + limit: usize, + ) -> Result>, PersistenceError> { + self.dirty_query_projection_entity_ids(tenant, entity_type, limit) + .await + .map(Some) + } } #[async_trait::async_trait] @@ -199,6 +278,30 @@ impl QueryPlaneStore for TursoEventStore { .await } + async fn upsert_projection_if_source( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + status: &str, + fields: &serde_json::Value, + state: &serde_json::Value, + sequence_nr: u64, + source: ProjectionSourceFence<'_>, + ) -> Result { + self.upsert_query_projection_with_state_if_source( + tenant, + entity_type, + entity_id, + status, + fields, + state, + sequence_nr, + source, + ) + .await + } + async fn upsert_projections( &self, tenant: &str, @@ -222,6 +325,50 @@ impl QueryPlaneStore for TursoEventStore { .await } + async fn remove_projection_if_source( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + source: ProjectionSourceFence<'_>, + ) -> Result { + self.remove_query_projection_if_source(tenant, entity_type, entity_id, source) + .await + } + + async fn clear_projection_dirty_if_source( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + source: ProjectionSourceFence<'_>, + ) -> Result { + self.clear_query_projection_dirty_if_source(tenant, entity_type, entity_id, source) + .await + } + + async fn remove_projection_if_exact( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + status: &str, + fields: &serde_json::Value, + state: &serde_json::Value, + sequence_nr: u64, + ) -> Result { + self.remove_query_projection_if_exact( + tenant, + entity_type, + entity_id, + status, + fields, + state, + sequence_nr, + ) + .await + } + async fn query_field_index( &self, tenant: &str, @@ -316,172 +463,22 @@ impl QueryPlaneStore for TursoEventStore { .await .map(Some) } -} -#[async_trait::async_trait] -impl QueryPlaneStore for TenantStoreRouter { - async fn upsert_projection( + async fn dirty_projection_entity_ids( &self, tenant: &str, entity_type: &str, - entity_id: &str, - status: &str, - fields: &serde_json::Value, - state: &serde_json::Value, - sequence_nr: u64, - ) -> Result<(), PersistenceError> { - let store = self.store_for_tenant(tenant).await?; - store - .upsert_query_projection_with_state( - tenant, - entity_type, - entity_id, - status, - fields, - state, - sequence_nr, - ) - .await - } - - async fn upsert_projections( - &self, - tenant: &str, - projections: &[QueryProjectionUpsert], - ) -> Result<(), PersistenceError> { - let store = self.store_for_tenant(tenant).await?; - let turso_projections = projections - .iter() - .map(to_turso_projection) - .collect::>(); - store - .upsert_query_projections(tenant, &turso_projections) - .await - } - - async fn remove_projection( - &self, - tenant: &str, - entity_type: &str, - entity_id: &str, - ) -> Result<(), PersistenceError> { - let store = self.store_for_tenant(tenant).await?; - store - .remove_query_projection(tenant, entity_type, entity_id) - .await - } - - async fn query_field_index( - &self, - tenant: &str, - entity_type: &str, - where_clause: &str, - params: Vec, + limit: usize, ) -> Result>, PersistenceError> { - let store = self.store_for_tenant(tenant).await?; - TursoEventStore::query_field_index(&store, tenant, entity_type, where_clause, params) + self.dirty_query_projection_entity_ids(tenant, entity_type, limit) .await .map(Some) } - - async fn query_field_index_page( - &self, - tenant: &str, - entity_type: &str, - where_clause: &str, - params: Vec, - order_by: &[QueryFieldIndexOrder], - skip: usize, - top: usize, - include_count: bool, - ) -> Result, PersistenceError> { - let store = self.store_for_tenant(tenant).await?; - let order_by = storage_order_by(order_by); - let (entity_ids, total_count) = store - .query_field_index_page( - tenant, - entity_type, - where_clause, - params, - &order_by, - skip, - top, - include_count, - ) - .await?; - Ok(Some(QueryFieldIndexPage { - entity_ids, - total_count, - })) - } - - async fn load_projection_fields_many( - &self, - tenant: &str, - entity_type: &str, - entity_ids: &[String], - field_names: &[&str], - ) -> Result>, PersistenceError> { - let store = self.store_for_tenant(tenant).await?; - store - .load_query_projection_fields_many(tenant, entity_type, entity_ids, field_names) - .await - .map(|rows| { - Some( - rows.into_iter() - .map(|row| QueryProjectionFieldsRow { - entity_id: row.entity_id, - status: row.status, - fields: row.fields, - }) - .collect(), - ) - }) - } - - async fn load_entity_catalog_rows( - &self, - tenant: &str, - entity_type: &str, - entity_ids: &[String], - ) -> Result>, PersistenceError> { - let store = self.store_for_tenant(tenant).await?; - TursoEventStore::load_entity_catalog_rows(&store, tenant, entity_type, entity_ids) - .await - .map(|rows| { - Some( - rows.into_iter() - .map(|row| EntityCatalogRow { - entity_id: row.entity_id, - status: row.status, - fields: row.fields, - state: row.state, - sequence_nr: row.sequence_nr, - }) - .collect(), - ) - }) - } - - async fn projected_entity_counts_by_tenant( - &self, - ) -> Result>, PersistenceError> { - let mut counts = Vec::new(); - for tenant_id in self.connected_tenants().await { - let store = self.store_for_tenant(&tenant_id).await?; - if let Some((_, count)) = TursoEventStore::projected_entity_counts_by_tenant(&store) - .await? - .into_iter() - .find(|(tenant, _)| tenant == &tenant_id) - { - counts.push((tenant_id, count)); - } - } - Ok(Some(counts)) - } } -fn to_turso_projection(projection: &QueryProjectionUpsert) -> TursoQueryProjectionUpsert { +pub(super) fn to_turso_projection( + projection: &QueryProjectionUpsert, +) -> TursoQueryProjectionUpsert { TursoQueryProjectionUpsert { entity_type: projection.entity_type.clone(), entity_id: projection.entity_id.clone(), diff --git a/crates/temper-server/src/storage/query_plane_impls/router.rs b/crates/temper-server/src/storage/query_plane_impls/router.rs new file mode 100644 index 000000000..fb7abbd2d --- /dev/null +++ b/crates/temper-server/src/storage/query_plane_impls/router.rs @@ -0,0 +1,255 @@ +use temper_store_turso::{TenantStoreRouter, TursoEventStore}; + +use super::*; + +#[async_trait::async_trait] +impl QueryPlaneStore for TenantStoreRouter { + async fn upsert_projection( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + status: &str, + fields: &serde_json::Value, + state: &serde_json::Value, + sequence_nr: u64, + ) -> Result<(), PersistenceError> { + let store = self.store_for_tenant(tenant).await?; + store + .upsert_query_projection_with_state( + tenant, + entity_type, + entity_id, + status, + fields, + state, + sequence_nr, + ) + .await + } + + async fn upsert_projection_if_source( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + status: &str, + fields: &serde_json::Value, + state: &serde_json::Value, + sequence_nr: u64, + source: ProjectionSourceFence<'_>, + ) -> Result { + let store = self.store_for_tenant(tenant).await?; + store + .upsert_query_projection_with_state_if_source( + tenant, + entity_type, + entity_id, + status, + fields, + state, + sequence_nr, + source, + ) + .await + } + + async fn upsert_projections( + &self, + tenant: &str, + projections: &[QueryProjectionUpsert], + ) -> Result<(), PersistenceError> { + let store = self.store_for_tenant(tenant).await?; + let turso_projections = projections + .iter() + .map(to_turso_projection) + .collect::>(); + store + .upsert_query_projections(tenant, &turso_projections) + .await + } + + async fn remove_projection( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + ) -> Result<(), PersistenceError> { + let store = self.store_for_tenant(tenant).await?; + store + .remove_query_projection(tenant, entity_type, entity_id) + .await + } + + async fn remove_projection_if_source( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + source: ProjectionSourceFence<'_>, + ) -> Result { + let store = self.store_for_tenant(tenant).await?; + store + .remove_query_projection_if_source(tenant, entity_type, entity_id, source) + .await + } + + async fn clear_projection_dirty_if_source( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + source: ProjectionSourceFence<'_>, + ) -> Result { + let store = self.store_for_tenant(tenant).await?; + store + .clear_query_projection_dirty_if_source(tenant, entity_type, entity_id, source) + .await + } + + async fn remove_projection_if_exact( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + status: &str, + fields: &serde_json::Value, + state: &serde_json::Value, + sequence_nr: u64, + ) -> Result { + let store = self.store_for_tenant(tenant).await?; + store + .remove_query_projection_if_exact( + tenant, + entity_type, + entity_id, + status, + fields, + state, + sequence_nr, + ) + .await + } + + async fn query_field_index( + &self, + tenant: &str, + entity_type: &str, + where_clause: &str, + params: Vec, + ) -> Result>, PersistenceError> { + let store = self.store_for_tenant(tenant).await?; + TursoEventStore::query_field_index(&store, tenant, entity_type, where_clause, params) + .await + .map(Some) + } + + async fn query_field_index_page( + &self, + tenant: &str, + entity_type: &str, + where_clause: &str, + params: Vec, + order_by: &[QueryFieldIndexOrder], + skip: usize, + top: usize, + include_count: bool, + ) -> Result, PersistenceError> { + let store = self.store_for_tenant(tenant).await?; + let order_by = storage_order_by(order_by); + let (entity_ids, total_count) = store + .query_field_index_page( + tenant, + entity_type, + where_clause, + params, + &order_by, + skip, + top, + include_count, + ) + .await?; + Ok(Some(QueryFieldIndexPage { + entity_ids, + total_count, + })) + } + + async fn load_projection_fields_many( + &self, + tenant: &str, + entity_type: &str, + entity_ids: &[String], + field_names: &[&str], + ) -> Result>, PersistenceError> { + let store = self.store_for_tenant(tenant).await?; + store + .load_query_projection_fields_many(tenant, entity_type, entity_ids, field_names) + .await + .map(|rows| { + Some( + rows.into_iter() + .map(|row| QueryProjectionFieldsRow { + entity_id: row.entity_id, + status: row.status, + fields: row.fields, + }) + .collect(), + ) + }) + } + + async fn load_entity_catalog_rows( + &self, + tenant: &str, + entity_type: &str, + entity_ids: &[String], + ) -> Result>, PersistenceError> { + let store = self.store_for_tenant(tenant).await?; + TursoEventStore::load_entity_catalog_rows(&store, tenant, entity_type, entity_ids) + .await + .map(|rows| { + Some( + rows.into_iter() + .map(|row| EntityCatalogRow { + entity_id: row.entity_id, + status: row.status, + fields: row.fields, + state: row.state, + sequence_nr: row.sequence_nr, + }) + .collect(), + ) + }) + } + + async fn projected_entity_counts_by_tenant( + &self, + ) -> Result>, PersistenceError> { + let mut counts = Vec::new(); + for tenant_id in self.connected_tenants().await { + let store = self.store_for_tenant(&tenant_id).await?; + if let Some((_, count)) = TursoEventStore::projected_entity_counts_by_tenant(&store) + .await? + .into_iter() + .find(|(tenant, _)| tenant == &tenant_id) + { + counts.push((tenant_id, count)); + } + } + Ok(Some(counts)) + } + + async fn dirty_projection_entity_ids( + &self, + tenant: &str, + entity_type: &str, + limit: usize, + ) -> Result>, PersistenceError> { + let store = self.store_for_tenant(tenant).await?; + store + .dirty_query_projection_entity_ids(tenant, entity_type, limit) + .await + .map(Some) + } +} diff --git a/crates/temper-server/src/webhooks/receiver.rs b/crates/temper-server/src/webhooks/receiver.rs index bda797289..a9b84516b 100644 --- a/crates/temper-server/src/webhooks/receiver.rs +++ b/crates/temper-server/src/webhooks/receiver.rs @@ -38,6 +38,18 @@ pub async fn handle_webhook( Query(query): Query>, ) -> impl IntoResponse { let tenant = TenantId::new(&tenant_str); + let Some(_generation) = state.try_begin_tenant_request(&tenant).await else { + return ( + StatusCode::SERVICE_UNAVAILABLE, + "Tenant runtime generation is being published; retry the webhook".to_string(), + ); + }; + if state.spec_publication_gated(&tenant) { + return ( + StatusCode::SERVICE_UNAVAILABLE, + "Tenant runtime generation is being published; retry the webhook".to_string(), + ); + } // Look up webhook config from registry. let lookup = find_webhook(&state, &tenant, &webhook_path); diff --git a/crates/temper-server/tests/common/platform_harness.rs b/crates/temper-server/tests/common/platform_harness.rs index 48edbd9bb..c02f0a726 100644 --- a/crates/temper-server/tests/common/platform_harness.rs +++ b/crates/temper-server/tests/common/platform_harness.rs @@ -94,13 +94,20 @@ impl SimPlatformHarness { pub fn register_inline_spec(&self, tenant: &str, entity_type: &str, ioa_source: &str) { let automaton = temper_spec::automaton::parse_automaton(ioa_source).expect("inline IOA should parse"); - let table = temper_jit::table::TransitionTable::from_automaton(&automaton); + let mut table = temper_jit::table::TransitionTable::from_automaton(&automaton); let mut registry = self.platform_state.server.registry.write().unwrap(); // ci-ok: infallible lock let spec = registry .get_spec_mut(&TenantId::new(tenant), entity_type) .unwrap_or_else(|| { panic!("entity type '{entity_type}' not found for tenant '{tenant}'") }); + let current_table = spec.table(); + assert_eq!( + temper_server::key_index::declared_key_set_signature(&table.keys), + temper_server::key_index::declared_key_set_signature(¤t_table.keys), + "inline spec overrides cannot change an activated declared-key contract" + ); + table.key_contract_activation_epoch = current_table.key_contract_activation_epoch; spec.swap_controller().swap(table); spec.integrations = automaton.integrations; spec.ioa_source = ioa_source.to_string(); diff --git a/crates/temper-server/tests/custom_effect_retry.rs b/crates/temper-server/tests/custom_effect_retry.rs new file mode 100644 index 000000000..c0b091371 --- /dev/null +++ b/crates/temper-server/tests/custom_effect_retry.rs @@ -0,0 +1,125 @@ +//! Durable retry coverage for post-dispatch platform effects. + +mod common; + +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use temper_runtime::scheduler::install_deterministic_context; +use temper_runtime::tenant::TenantId; +use temper_server::request_context::AgentContext; +use temper_server::state::custom_effects::CustomEffectHandler; +use temper_server::{ServerState, state::custom_effects}; +use temper_store_sim::SimEventStore; + +const EFFECT_SPEC: &str = r#" +[automaton] +name = "EffectOwner" +states = ["Ready"] +initial = "Ready" + +[[action]] +name = "Publish" +kind = "input" +from = ["Ready"] +to = "Ready" +params = ["Value"] +effect = "trigger DurablePublish" +"#; + +struct FailOnceEffect { + attempts: AtomicUsize, +} + +#[async_trait::async_trait] +impl CustomEffectHandler for FailOnceEffect { + async fn handle( + &self, + effect_name: &str, + _entity_type: &str, + _entity_id: &str, + _entity_fields: &serde_json::Value, + _server: &ServerState, + ) -> Result<(), String> { + assert_eq!(effect_name, "DurablePublish"); + let attempt = self.attempts.fetch_add(1, Ordering::SeqCst) + 1; + if attempt == 1 { + Err("injected publication failure".to_string()) + } else { + Ok(()) + } + } +} + +fn state_with_handler( + store: SimEventStore, + system_name: &str, + handler: Arc, +) -> ServerState { + let mut state = common::build_single_tenant_state_with_store( + store, + system_name, + "default", + &[("EffectOwner", EFFECT_SPEC)], + ); + let handler: Arc = handler; + state.custom_effect_handler = Some(handler); + state +} + +#[tokio::test] +async fn failed_custom_effect_replays_exactly_after_process_restart() { + let (_guard, _clock, _ids) = install_deterministic_context(303); + let store = SimEventStore::no_faults(303); + let handler = Arc::new(FailOnceEffect { + attempts: AtomicUsize::new(0), + }); + let tenant = TenantId::default(); + let mut agent = AgentContext::for_service("effect-retry-test"); + agent.idempotency_key = Some("publish-once".to_string()); + + let first_state = state_with_handler(store.clone(), "effect-first", Arc::clone(&handler)); + let first = first_state + .dispatch_tenant_action( + &tenant, + "EffectOwner", + "owner-1", + "Publish", + serde_json::json!({"Value": "v1"}), + &agent, + ) + .await + .expect("durable action returns its effect outcome"); + assert!( + !first.success, + "a committed action must remain retryable while its durable custom effect failed" + ); + let committed_sequence = first.state.sequence_nr; + assert!(committed_sequence > 0); + assert_eq!(handler.attempts.load(Ordering::SeqCst), 1); + drop(first_state); + + let restarted = state_with_handler(store, "effect-restart", Arc::clone(&handler)); + let retried = restarted + .dispatch_tenant_action( + &tenant, + "EffectOwner", + "owner-1", + "Publish", + serde_json::json!({"Value": "v1"}), + &agent, + ) + .await + .expect("retry recovers the committed transition"); + + assert!(retried.success, "the recovered effect retry must complete"); + assert_eq!( + retried.state.sequence_nr, committed_sequence, + "effect retry must not append the domain action again" + ); + assert_eq!( + handler.attempts.load(Ordering::SeqCst), + 2, + "the durable idempotency record must replay the original effect after restart" + ); +} diff --git a/crates/temper-server/tests/dispatch_retry_idempotency.rs b/crates/temper-server/tests/dispatch_retry_idempotency.rs index f8dbb5209..360732c44 100644 --- a/crates/temper-server/tests/dispatch_retry_idempotency.rs +++ b/crates/temper-server/tests/dispatch_retry_idempotency.rs @@ -57,7 +57,7 @@ fn build_state_with_sim_store(seed: u64) -> (ServerState, SimEventStore) { (state, sim_store) } -#[tokio::test] +#[tokio::test(start_paused = true)] async fn retry_after_dropped_reply_replays_success_response_and_runs_effects_without_header() { let (_guard, _clock, _ids) = install_deterministic_context(48); let (state, sim_store) = build_state_with_sim_store(48); @@ -75,19 +75,24 @@ async fn retry_after_dropped_reply_replays_success_response_and_runs_effects_wit .await .expect("entity creation succeeds"); - sim_store.inject_append_delay(&persistence_id, Duration::from_millis(25)); - - let response = state - .dispatch_tenant_action( - &tenant, - "TimedTask", - entity_id, - "Start", - serde_json::json!({}), - &AgentContext::default(), - ) - .await - .expect("dispatch should recover the timed-out first reply"); + let pause = sim_store.inject_postcommit_append_pause(&persistence_id); + let agent_context = AgentContext::default(); + let dispatch = state.dispatch_tenant_action( + &tenant, + "TimedTask", + entity_id, + "Start", + serde_json::json!({}), + &agent_context, + ); + let advance_past_timeout = async { + pause.wait_until_reached().await; + tokio::time::advance(Duration::from_millis(6)).await; + pause.resume(); + tokio::time::advance(Duration::from_secs(1)).await; + }; + let (response, ()) = tokio::join!(dispatch, advance_past_timeout); + let response = response.expect("dispatch should recover the timed-out first reply"); assert!( response.success, @@ -102,3 +107,51 @@ async fn retry_after_dropped_reply_replays_success_response_and_runs_effects_wit "post-dispatch effects from the successful Start transition must arm the state_timeout" ); } + +#[tokio::test(start_paused = true)] +async fn field_update_retry_after_post_commit_timeout_appends_once() { + let (_guard, _clock, _ids) = install_deterministic_context(238); + let (state, sim_store) = build_state_with_sim_store(238); + let tenant = TenantId::default(); + let entity_id = "timed-task-field-update"; + let persistence_id = format!("default:TimedTask:{entity_id}"); + + state + .get_or_create_tenant_entity( + &tenant, + "TimedTask", + entity_id, + serde_json::json!({"Id": entity_id, "Title": "before"}), + ) + .await + .expect("entity creation succeeds"); + + // The sim store commits before reaching this one-shot barrier. The first ask + // therefore times out after durability but before the actor can reply; the + // dispatch retry must reuse one token and observe the committed update. + let pause = sim_store.inject_postcommit_append_pause(&persistence_id); + let update = state.update_tenant_entity_fields( + &tenant, + "TimedTask", + entity_id, + serde_json::json!({"Title": "after"}), + false, + Some("patch-post-commit-timeout".to_string()), + ); + let advance_past_timeout = async { + pause.wait_until_reached().await; + tokio::time::advance(Duration::from_millis(6)).await; + pause.resume(); + tokio::time::advance(Duration::from_secs(1)).await; + }; + let (response, ()) = tokio::join!(update, advance_past_timeout); + let response = response.expect("retry should return the committed field update"); + + assert!(response.success); + assert_eq!(response.state.fields["Title"], "after"); + assert_eq!( + sim_store.dump_journal(&persistence_id).len(), + 2, + "Create plus exactly one private field-update event must be durable" + ); +} diff --git a/crates/temper-server/tests/dst_concurrency_retry.rs b/crates/temper-server/tests/dst_concurrency_retry.rs index e527ddc60..c8c995d5d 100644 --- a/crates/temper-server/tests/dst_concurrency_retry.rs +++ b/crates/temper-server/tests/dst_concurrency_retry.rs @@ -16,6 +16,7 @@ use std::time::Duration; use temper_jit::table::TransitionTable; use temper_runtime::ActorSystem; +use temper_runtime::persistence::EventStore; use temper_runtime::scheduler::install_deterministic_context; use temper_server::storage::{BackendLabel, BoxedEventStore}; use temper_server::{EntityActor, EntityMsg, EntityResponse}; @@ -52,6 +53,26 @@ async fn dispatch_action( .expect("actor should respond") } +async fn dispatch_action_with_key( + actor_ref: &temper_runtime::actor::ActorRef, + action: &str, + params: serde_json::Value, + idempotency_key: &str, +) -> EntityResponse { + actor_ref + .ask( + EntityMsg::Action { + name: action.to_string(), + params, + cross_entity_booleans: BTreeMap::new(), + idempotency_key: Some(idempotency_key.to_string()), + }, + Duration::from_secs(5), + ) + .await + .expect("actor should respond") +} + /// Wait for the actor's `pre_start` to complete (which writes the bootstrap /// `Create` event to the journal) before injecting retry faults on later /// appends. Using `GetState` as the synchronisation point — it only replies @@ -134,6 +155,113 @@ async fn dst_retry_succeeds_after_one_violation() { assert_eq!(sim.dump_journal(&persistence_id).len(), 2); } +/// Catch-up must rebuild from a clean initial state. Replaying the existing +/// pre-race actor state would apply the first non-idempotent `AddItem` twice. +#[tokio::test] +async fn dst_retry_does_not_reapply_preexisting_effects() { + let seed = 8; + let (_guard, _clock, _id_gen) = install_deterministic_context(seed); + let (store, sim) = sim_store_with_handle(seed); + let table = order_table(); + let entity_id = "ord-retry-existing-effect"; + let persistence_id = format!("default:Order:{entity_id}"); + + let system = ActorSystem::new("dst-retry-existing-effect"); + let actor_ref = spawn_order(&system, table, store, entity_id); + wait_ready(&actor_ref).await; + let first = dispatch_action(&actor_ref, "AddItem", serde_json::json!({})).await; + assert!(first.success); + assert_eq!(first.state.item_count, 1); + + sim.inject_concurrency_violations(&persistence_id, 1); + let retried = dispatch_action(&actor_ref, "AddItem", serde_json::json!({})).await; + + assert!(retried.success, "retry failed: {:?}", retried.error); + assert_eq!( + retried.state.item_count, 2, + "the preexisting AddItem effect must be replayed exactly once" + ); + assert_eq!(sim.dump_journal(&persistence_id).len(), 3); +} + +/// Two stale actor replicas can race the first journal write from a +/// snapshot-only generation. The loser must treat the idempotency key recovered +/// from the winner as the committed result, not append the action again. +#[tokio::test] +async fn dst_snapshot_handoff_retry_deduplicates_the_winning_action() { + let seed = 289; + let (_guard, _clock, _id_gen) = install_deterministic_context(seed); + let (store, sim) = sim_store_with_handle(seed); + let entity_id = "snapshot-handoff-idempotency-race"; + let persistence_id = format!("default:Order:{entity_id}"); + let snapshot = serde_json::to_vec(&serde_json::json!({ + "entity_type": "Order", + "entity_id": entity_id, + "status": "Draft", + "item_count": 0, + "counters": {}, + "booleans": {}, + "lists": {}, + "fields": {"Id": entity_id, "Status": "Draft"}, + "events": [], + "total_event_count": 0, + "events_since_snapshot": 0, + "last_snapshot_sequence_nr": 5, + "sequence_nr": 5, + "processed_idempotency_keys": {} + })) + .expect("serialize snapshot-only source"); + sim.save_snapshot(&persistence_id, 5, &snapshot) + .await + .expect("seed snapshot-only source"); + + let system = ActorSystem::new("dst-snapshot-handoff-idempotency-race"); + let first = spawn_order(&system, order_table(), store.clone(), entity_id); + let second = system.spawn( + EntityActor::with_persistence( + "Order", + entity_id, + order_table(), + serde_json::json!({}), + store, + BackendLabel::Sim, + ) + .with_tenant("default"), + "snapshot-handoff-idempotency-race-replica", + ); + wait_ready(&first).await; + wait_ready(&second).await; + + let winner = dispatch_action_with_key( + &first, + "AddItem", + serde_json::json!({}), + "same-first-journal-action", + ) + .await; + assert!(winner.success); + assert_eq!(winner.state.item_count, 1); + + let loser = dispatch_action_with_key( + &second, + "AddItem", + serde_json::json!({}), + "same-first-journal-action", + ) + .await; + assert!(loser.success); + assert_eq!(loser.state.item_count, 1); + assert_eq!(loser.state.total_event_count, 1); + assert_eq!( + sim.dump_journal(&persistence_id) + .iter() + .map(|event| event.event_type.as_str()) + .collect::>(), + vec!["Temper.Internal.StateMaterialization.v1", "AddItem"], + "the stale replica must not append the winner's idempotent action twice" + ); +} + // ------------------------------------------------------------------------- // Test 2: Retry budget exhausts cleanly under sustained violation. // ------------------------------------------------------------------------- diff --git a/crates/temper-server/tests/dst_entity_key_index.rs b/crates/temper-server/tests/dst_entity_key_index.rs index 07dc0a66c..0a65ccc53 100644 --- a/crates/temper-server/tests/dst_entity_key_index.rs +++ b/crates/temper-server/tests/dst_entity_key_index.rs @@ -16,7 +16,9 @@ use std::time::Duration; use temper_jit::table::TransitionTable; use temper_runtime::ActorSystem; -use temper_runtime::persistence::{EntityKeyRow, EventMetadata, EventStore, PersistenceEnvelope}; +use temper_runtime::persistence::{ + EntityKeyRow, EventMetadata, EventStore, KeyIndexBackfillFence, PersistenceEnvelope, +}; use temper_runtime::scheduler::{install_deterministic_context, sim_now, sim_uuid}; use temper_server::key_index::canonical_key_hash; use temper_server::storage::{BackendLabel, BoxedEventStore}; @@ -49,10 +51,28 @@ async fn dispatch( .expect("actor should respond") } +async fn delete(actor_ref: &temper_runtime::actor::ActorRef) -> EntityResponse { + actor_ref + .ask(EntityMsg::Delete, Duration::from_secs(5)) + .await + .expect("actor should respond") +} + +async fn get_state(actor_ref: &temper_runtime::actor::ActorRef) -> EntityResponse { + actor_ref + .ask(EntityMsg::GetState, Duration::from_secs(5)) + .await + .expect("actor should respond") +} + fn doc_key_hash(workspace: &str, path: &str) -> String { + doc_key_hash_values(serde_json::json!(workspace), serde_json::json!(path)) +} + +fn doc_key_hash_values(workspace: serde_json::Value, path: serde_json::Value) -> String { let mut fields = serde_json::Map::new(); - fields.insert("WorkspaceId".to_string(), serde_json::json!(workspace)); - fields.insert("Path".to_string(), serde_json::json!(path)); + fields.insert("WorkspaceId".to_string(), workspace); + fields.insert("Path".to_string(), path); canonical_key_hash( "path", &["WorkspaceId".to_string(), "Path".to_string()], @@ -112,6 +132,337 @@ async fn dst_keyed_read_is_present_iff_entity_exists() { } } +/// Deletion must release every declared key in the same atomic commit as the +/// tombstone. Otherwise the deleted entity remains the durable owner and a new +/// entity cannot reclaim the logically vacant key. +#[tokio::test] +async fn dst_delete_releases_declared_key_for_reclaim() { + for seed in 0..NUM_SEEDS { + let (_guard, _clock, _id) = install_deterministic_context(seed); + let store: BoxedEventStore = BoxedEventStore::new(SimEventStore::no_faults(seed)); + let table = doc_table(); + let key_hash = doc_key_hash("ws1", "/reclaim.md"); + + let system = ActorSystem::new("dst-key-reclaim"); + let first_id = format!("doc-first-{seed}"); + let first = EntityActor::with_persistence( + "Doc", + &first_id, + table.clone(), + serde_json::json!({}), + store.clone(), + BackendLabel::Sim, + ) + .with_tenant("default"); + let first_ref = system.spawn(first, &first_id); + + let created = dispatch( + &first_ref, + "Create", + serde_json::json!({ "WorkspaceId": "ws1", "Path": "/reclaim.md" }), + ) + .await; + assert!( + created.success, + "seed {seed}: first Create failed: {:?}", + created.error + ); + + let deleted = delete(&first_ref).await; + assert!( + deleted.success, + "seed {seed}: Delete failed: {:?}", + deleted.error + ); + assert_eq!( + store + .lookup_by_key("default", "Doc", "path", &key_hash) + .await + .expect("lookup after delete"), + None, + "seed {seed}: a tombstoned entity must release its declared key" + ); + + drop(first_ref); + drop(system); + + // Recreate the actor system and recover the deleted stream. Tombstone state + // and released ownership must both survive replay, not just the live actor. + let restarted = ActorSystem::new("dst-key-reclaim-restarted"); + let recovered = EntityActor::with_persistence( + "Doc", + &first_id, + table.clone(), + serde_json::json!({}), + store.clone(), + BackendLabel::Sim, + ) + .with_tenant("default"); + let recovered_ref = restarted.spawn(recovered, &first_id); + let recovered_state = get_state(&recovered_ref).await; + assert_eq!(recovered_state.state.status, "Deleted"); + assert_eq!( + store + .lookup_by_key("default", "Doc", "path", &key_hash) + .await + .expect("lookup after recovery"), + None, + "seed {seed}: replay must not restore tombstoned ownership" + ); + + // Two replacements race for the now-vacant key. The store must serialize + // uniqueness so exactly one journal advances beyond its bootstrap event. + let left_id = format!("doc-left-{seed}"); + let right_id = format!("doc-right-{seed}"); + let spawn_replacement = |entity_id: &str| { + restarted.spawn( + EntityActor::with_persistence( + "Doc", + entity_id, + table.clone(), + serde_json::json!({}), + store.clone(), + BackendLabel::Sim, + ) + .with_tenant("default"), + entity_id, + ) + }; + let left = spawn_replacement(&left_id); + let right = spawn_replacement(&right_id); + let params = serde_json::json!({ "WorkspaceId": "ws1", "Path": "/reclaim.md" }); + let (left_result, right_result) = tokio::join!( + dispatch(&left, "Create", params.clone()), + dispatch(&right, "Create", params) + ); + assert_ne!(left_result.success, right_result.success); + let (winner, loser) = if left_result.success { + (&left_id, &right_id) + } else { + (&right_id, &left_id) + }; + assert_eq!( + store + .lookup_by_key("default", "Doc", "path", &key_hash) + .await + .expect("lookup after reclaim"), + Some(winner.clone()), + "seed {seed}: the reclaimed key must resolve to the live replacement" + ); + assert_eq!( + store + .read_events(&format!("default:Doc:{loser}"), 0) + .await + .unwrap() + .len(), + 1, + "seed {seed}: losing journal must retain only its bootstrap event" + ); + } +} + +/// Retried delivery after a successful tombstone append is idempotent. A duplicate +/// terminal suffix would make replay stop below the store's durable high-water and +/// leave exact repair unable to publish coverage. +#[tokio::test] +async fn dst_retried_delete_does_not_append_a_terminal_suffix() { + let seed = 238; + let (_guard, _clock, _id) = install_deterministic_context(seed); + let sim = SimEventStore::no_faults(seed); + let store = BoxedEventStore::new(sim.clone()); + let table = doc_table(); + let entity_id = "doc-retried-delete"; + let persistence_id = format!("default:Doc:{entity_id}"); + let system = ActorSystem::new("dst-retried-delete"); + let actor_ref = system.spawn( + EntityActor::with_persistence( + "Doc", + entity_id, + table.clone(), + serde_json::json!({}), + store.clone(), + BackendLabel::Sim, + ) + .with_tenant("default"), + entity_id, + ); + assert!( + dispatch( + &actor_ref, + "Create", + serde_json::json!({"WorkspaceId": "ws", "Path": "/retry-delete"}), + ) + .await + .success + ); + let sequence_before_delete = sim + .dump_journal(&persistence_id) + .last() + .expect("created history") + .sequence_nr; + let first_delete = delete(&actor_ref).await; + assert!(first_delete.success); + assert_eq!(first_delete.state.sequence_nr, sequence_before_delete + 1); + let second_delete = delete(&actor_ref).await; + assert!( + second_delete.success, + "retry should be an idempotent success" + ); + assert_eq!( + sim.dump_journal(&persistence_id).len(), + (sequence_before_delete + 1) as usize, + "the durable prefix plus exactly one terminal event" + ); + drop(actor_ref); + drop(system); + + let restarted_system = ActorSystem::new("dst-retried-delete-restart"); + let restarted = restarted_system.spawn( + EntityActor::with_persistence( + "Doc", + entity_id, + table, + serde_json::json!({}), + store.clone(), + BackendLabel::Sim, + ) + .with_tenant("default"), + "doc-retried-delete-restarted", + ); + let recovered = get_state(&restarted).await.state; + assert_eq!(recovered.status, "Deleted"); + assert_eq!(recovered.sequence_nr, sequence_before_delete + 1); + + let signature = "v4:path"; + let revision = store + .begin_key_index_backfill("default", "Doc", signature) + .await + .expect("begin exact repair"); + store + .backfill_entity_keys( + "default", + "Doc", + entity_id, + recovered.sequence_nr, + KeyIndexBackfillFence { + key_set_signature: signature, + contract_revision: revision, + expected_journal_sequence: recovered.sequence_nr, + expected_entity_live: false, + expected_snapshot: None, + }, + &[], + ) + .await + .expect("deleted stream must remain repairable"); + assert!( + store + .mark_key_index_backfilled_if_revision("default", "Doc", signature, revision) + .await + .expect("publish repaired coverage") + ); +} + +/// Every persisted re-key replaces the entity's complete ownership set: old +/// values disappear, partial-null composite values remain indexable, and an +/// all-null composite releases ownership rather than leaving its prior row. +#[tokio::test] +async fn dst_rekey_reconciles_rename_partial_null_and_all_null() { + for seed in 0..NUM_SEEDS { + let (_guard, _clock, _id) = install_deterministic_context(seed); + let store: BoxedEventStore = BoxedEventStore::new(SimEventStore::no_faults(seed)); + let table = doc_table(); + let entity_id = format!("doc-rekey-{seed}"); + let system = ActorSystem::new("dst-key-rekey"); + let actor = EntityActor::with_persistence( + "Doc", + &entity_id, + table, + serde_json::json!({}), + store.clone(), + BackendLabel::Sim, + ) + .with_tenant("default"); + let actor_ref = system.spawn(actor, &entity_id); + + let created = dispatch( + &actor_ref, + "Create", + serde_json::json!({ "WorkspaceId": "ws1", "Path": "/old.md" }), + ) + .await; + assert!(created.success, "seed {seed}: Create failed"); + + let renamed = dispatch( + &actor_ref, + "Rekey", + serde_json::json!({ "WorkspaceId": "ws1", "Path": "/new.md" }), + ) + .await; + assert!(renamed.success, "seed {seed}: string rename failed"); + assert_eq!( + store + .lookup_by_key("default", "Doc", "path", &doc_key_hash("ws1", "/old.md")) + .await + .unwrap(), + None, + "seed {seed}: rename must release the old key" + ); + assert_eq!( + store + .lookup_by_key("default", "Doc", "path", &doc_key_hash("ws1", "/new.md")) + .await + .unwrap(), + Some(entity_id.clone()), + "seed {seed}: rename must claim the new key" + ); + + let partial_null = dispatch( + &actor_ref, + "Rekey", + serde_json::json!({ "WorkspaceId": "ws1", "Path": null }), + ) + .await; + assert!( + partial_null.success, + "seed {seed}: partial-null re-key failed" + ); + let partial_hash = doc_key_hash_values(serde_json::json!("ws1"), serde_json::Value::Null); + assert_eq!( + store + .lookup_by_key("default", "Doc", "path", &doc_key_hash("ws1", "/new.md")) + .await + .unwrap(), + None, + "seed {seed}: partial-null re-key must release the prior string key" + ); + assert_eq!( + store + .lookup_by_key("default", "Doc", "path", &partial_hash) + .await + .unwrap(), + Some(entity_id.clone()), + "seed {seed}: partial-null composite key remains indexable" + ); + + let all_null = dispatch( + &actor_ref, + "Rekey", + serde_json::json!({ "WorkspaceId": null, "Path": null }), + ) + .await; + assert!(all_null.success, "seed {seed}: all-null re-key failed"); + assert_eq!( + store + .lookup_by_key("default", "Doc", "path", &partial_hash) + .await + .unwrap(), + None, + "seed {seed}: all-null key must release the entity's final claim" + ); + } +} + /// Backfill (ADR-0153): an entity written BEFORE its key was declared has no key /// row, so a keyed read misses (would fall back to scan). After /// `backfill_entity_keys`, the keyed read resolves it — the path that lets a keyed @@ -143,13 +494,44 @@ async fn dst_backfill_makes_pre_existing_entity_keyed_findable() { key_name: "path".to_string(), key_hash: key_hash.clone(), }]; + let repair_signature = "v3:path"; + let repair_revision = store + .begin_key_index_backfill("default", "Doc", repair_signature) + .await + .unwrap(); store - .backfill_entity_keys("default", "Doc", &format!("doc-pre-{seed}"), &rows) + .backfill_entity_keys( + "default", + "Doc", + &format!("doc-pre-{seed}"), + 1, + KeyIndexBackfillFence { + key_set_signature: repair_signature, + contract_revision: repair_revision, + expected_journal_sequence: 1, + expected_entity_live: true, + expected_snapshot: None, + }, + &rows, + ) .await .unwrap(); // Idempotent: a second backfill is a no-op-equivalent. store - .backfill_entity_keys("default", "Doc", &format!("doc-pre-{seed}"), &rows) + .backfill_entity_keys( + "default", + "Doc", + &format!("doc-pre-{seed}"), + 1, + KeyIndexBackfillFence { + key_set_signature: repair_signature, + contract_revision: repair_revision, + expected_journal_sequence: 1, + expected_entity_live: true, + expected_snapshot: None, + }, + &rows, + ) .await .unwrap(); diff --git a/crates/temper-server/tests/dst_entity_key_reconciliation.rs b/crates/temper-server/tests/dst_entity_key_reconciliation.rs new file mode 100644 index 000000000..6d38436c1 --- /dev/null +++ b/crates/temper-server/tests/dst_entity_key_reconciliation.rs @@ -0,0 +1,708 @@ +//! DST for ADR-0192 exact declared-key reconciliation under failure and retry. + +use temper_runtime::persistence::{ + EntityKeyRow, EventMetadata, EventStore, IndexReconciliation, KeyIndexBackfillFence, + PersistenceEnvelope, PersistenceError, SnapshotBackfillFence, +}; +use temper_runtime::scheduler::{install_deterministic_context, sim_now, sim_uuid}; +use temper_store_sim::SimEventStore; + +#[path = "dst_entity_key_reconciliation/seeded_workload.rs"] +mod seeded_workload; + +fn envelope(event_type: &str) -> PersistenceEnvelope { + PersistenceEnvelope { + sequence_nr: 0, + event_type: event_type.to_string(), + payload: serde_json::json!({}), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: sim_now(), + actor_id: "dst-key-reconciliation".to_string(), + }, + } +} + +fn key(name: &str, hash: &str) -> EntityKeyRow { + EntityKeyRow { + key_name: name.to_string(), + key_hash: hash.to_string(), + } +} + +/// A key-only orphan can become live after repair enumeration without changing the +/// type contract. The exact repair must validate the older non-live classification +/// under the stream lock, not merely accept replay's now-current sequence. +#[tokio::test] +async fn dst_orphan_to_live_repair_is_liveness_fenced() { + for seed in [15] { + let (_guard, _clock, _id) = install_deterministic_context(seed); + let store = SimEventStore::no_faults(seed); + let persistence_id = "default:Doc:orphan-becomes-live"; + let live_key = key("path", "concurrent-live-path"); + let repair_signature = "v3:path"; + let repair_revision = store + .begin_key_index_backfill("default", "Doc", repair_signature) + .await + .expect("begin repair contract"); + store + .backfill_entity_keys( + "default", + "Doc", + "orphan-becomes-live", + 0, + KeyIndexBackfillFence { + key_set_signature: repair_signature, + contract_revision: repair_revision, + expected_journal_sequence: 0, + expected_entity_live: false, + expected_snapshot: None, + }, + std::slice::from_ref(&live_key), + ) + .await + .expect("seed key-only orphan"); + assert!( + store + .list_entity_ids_by_type("default", "Doc") + .await + .expect("classify live entities") + .is_empty(), + "seed {seed}: key-only repair candidate starts non-live" + ); + + store + .append_with_index_rows( + persistence_id, + 0, + &[envelope("Create")], + std::slice::from_ref(&live_key), + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(repair_signature.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .expect("same-contract create after repair enumeration"); + + let stale_repair = store + .backfill_entity_keys( + "default", + "Doc", + "orphan-becomes-live", + 1, + KeyIndexBackfillFence { + key_set_signature: repair_signature, + contract_revision: repair_revision, + expected_journal_sequence: 1, + expected_entity_live: false, + expected_snapshot: None, + }, + &[], + ) + .await; + assert!(matches!( + stale_repair, + Err(PersistenceError::EntityLivenessChanged { + expected_live: false, + actual_live: true, + }) + )); + assert_eq!( + store + .lookup_by_key("default", "Doc", "path", &live_key.key_hash) + .await + .expect("lookup concurrent live claim"), + Some("orphan-becomes-live".to_string()), + "seed {seed}: stale repair cannot delete the concurrent live claim" + ); + } +} + +/// A spec action does not need to be named `Deleted`: the persisted lifecycle +/// boundary is `to_status = "Deleted"`. A newer stale snapshot must not make that +/// terminal stream live again or allow its key to survive exact repair. +#[tokio::test] +async fn dst_action_named_tombstone_outranks_newer_snapshot() { + for seed in [16] { + let (_guard, _clock, _id) = install_deterministic_context(seed); + let store = SimEventStore::no_faults(seed); + let persistence_id = "default:Doc:action-named-delete"; + let stale_key = key("path", "deleted-path"); + let repair_signature = "v3:path"; + store + .append_with_index_rows( + persistence_id, + 0, + &[envelope("Create")], + std::slice::from_ref(&stale_key), + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(repair_signature.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .expect("create live owner"); + let mut tombstone = envelope("Delete"); + tombstone.payload = serde_json::json!({ + "action": "Delete", + "from_status": "Ready", + "to_status": "Deleted", + }); + store + .append_with_index_rows( + persistence_id, + 1, + &[tombstone], + &[], + &[], + IndexReconciliation { + keys: false, + key_set_signature: Some(repair_signature.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .expect("append action-named tombstone without key reconciliation"); + store + .save_snapshot(persistence_id, 3, b"newer-stale-live-snapshot") + .await + .expect("seed newer stale live snapshot"); + let repair_revision = store + .begin_key_index_backfill("default", "Doc", repair_signature) + .await + .expect("begin repair contract after derived writer fencing"); + + assert!( + store + .list_entity_ids_by_type("default", "Doc") + .await + .expect("classify live entities") + .is_empty(), + "seed {seed}: to_status tombstone outranks the newer snapshot" + ); + let stale_live_repair = store + .backfill_entity_keys( + "default", + "Doc", + "action-named-delete", + 3, + KeyIndexBackfillFence { + key_set_signature: repair_signature, + contract_revision: repair_revision, + expected_journal_sequence: 2, + expected_entity_live: true, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: 3, + state: b"newer-stale-live-snapshot", + }), + }, + &[], + ) + .await; + assert!(matches!( + stale_live_repair, + Err(PersistenceError::EntityLivenessChanged { + expected_live: true, + actual_live: false, + }) + )); + assert_eq!( + store + .lookup_by_key("default", "Doc", "path", &stale_key.key_hash) + .await + .expect("lookup rejected-repair owner"), + Some("action-named-delete".to_string()), + "seed {seed}: rejected repair is non-mutating" + ); + + store + .backfill_entity_keys( + "default", + "Doc", + "action-named-delete", + 2, + KeyIndexBackfillFence { + key_set_signature: repair_signature, + contract_revision: repair_revision, + expected_journal_sequence: 2, + expected_entity_live: false, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: 3, + state: b"newer-stale-live-snapshot", + }), + }, + &[], + ) + .await + .expect("purge action-named tombstone at the journal-owned durable sequence"); + assert_eq!( + store + .lookup_by_key("default", "Doc", "path", &stale_key.key_hash) + .await + .expect("lookup purged owner"), + None, + "seed {seed}: non-live repair removes stale ownership" + ); + } +} + +/// A rejected exact replacement changes neither journal nor ownership. Retrying the +/// same append replaces the complete set, including removal of a declaration no +/// longer emitted; a later empty exact set releases the remaining claim. +#[tokio::test] +async fn dst_failed_exact_reconcile_is_atomic_and_retryable() { + for seed in [11] { + let (_guard, _clock, _id) = install_deterministic_context(seed); + let store = SimEventStore::no_faults(seed); + let pid = "default:Doc:doc-a"; + let old_path = key("path", "old-path"); + let removed_decl = key("legacy", "legacy-value"); + let new_path = key("path", "new-path"); + + store + .append_with_keys( + pid, + 0, + &[envelope("Create")], + &[old_path.clone(), removed_decl.clone()], + ) + .await + .unwrap(); + + store.inject_concurrency_violations(pid, 1); + let rejected = store + .append_with_keys( + pid, + 1, + &[envelope("Rekey")], + std::slice::from_ref(&new_path), + ) + .await; + assert!( + matches!(rejected, Err(PersistenceError::ConcurrencyViolation { .. })), + "seed {seed}: injected pre-commit failure must reject the replacement" + ); + assert_eq!(store.read_events(pid, 0).await.unwrap().len(), 1); + for row in [&old_path, &removed_decl] { + assert_eq!( + store + .lookup_by_key("default", "Doc", &row.key_name, &row.key_hash) + .await + .unwrap(), + Some("doc-a".to_string()), + "seed {seed}: rejection must preserve the prior '{}' claim", + row.key_name + ); + } + + store + .append_with_keys( + pid, + 1, + &[envelope("Rekey")], + std::slice::from_ref(&new_path), + ) + .await + .unwrap(); + assert_eq!(store.read_events(pid, 0).await.unwrap().len(), 2); + for row in [&old_path, &removed_decl] { + assert_eq!( + store + .lookup_by_key("default", "Doc", &row.key_name, &row.key_hash) + .await + .unwrap(), + None, + "seed {seed}: retry must remove stale '{}' ownership", + row.key_name + ); + } + assert_eq!( + store + .lookup_by_key("default", "Doc", "path", "new-path") + .await + .unwrap(), + Some("doc-a".to_string()) + ); + + store + .append_with_keys(pid, 2, &[envelope("ClearKey")], &[]) + .await + .unwrap(); + assert_eq!( + store + .lookup_by_key("default", "Doc", "path", "new-path") + .await + .unwrap(), + None, + "seed {seed}: an authoritative empty set must release the final claim" + ); + } +} + +/// A background repair may replay sequence N while the live actor commits N+1. +/// The stale repair must be rejected rather than restoring N's old key row over the +/// newer rename. This is the deterministic interleaving history for the backfill +/// sequence fence; retrying from N+1 converges idempotently. +#[tokio::test] +async fn dst_stale_backfill_cannot_overwrite_newer_live_ownership() { + for seed in [12] { + let (_guard, _clock, _id) = install_deterministic_context(seed); + let store = SimEventStore::no_faults(seed); + let pid = "default:Doc:doc-race"; + let old_path = key("path", "old-path"); + let new_path = key("path", "new-path"); + + store + .append_with_keys( + pid, + 0, + &[envelope("Create")], + std::slice::from_ref(&old_path), + ) + .await + .unwrap(); + + // Backfill has replayed sequence 1 and derived old_path. Before it writes, + // the live actor commits a rename at sequence 2. + store + .append_with_keys( + pid, + 1, + &[envelope("Rekey")], + std::slice::from_ref(&new_path), + ) + .await + .unwrap(); + let repair_signature = "v3:path"; + let repair_revision = store + .begin_key_index_backfill("default", "Doc", repair_signature) + .await + .unwrap(); + let stale = store + .backfill_entity_keys( + "default", + "Doc", + "doc-race", + 1, + KeyIndexBackfillFence { + key_set_signature: repair_signature, + contract_revision: repair_revision, + expected_journal_sequence: 1, + expected_entity_live: true, + expected_snapshot: None, + }, + std::slice::from_ref(&old_path), + ) + .await; + assert!( + matches!( + stale, + Err(PersistenceError::JournalBoundaryChanged { + expected: 1, + actual: 2 + }) + ), + "seed {seed}: stale sequence-1 repair must be fenced after sequence 2" + ); + assert_eq!( + store + .lookup_by_key("default", "Doc", "path", "old-path") + .await + .unwrap(), + None, + "seed {seed}: stale ownership must not be restored" + ); + assert_eq!( + store + .lookup_by_key("default", "Doc", "path", "new-path") + .await + .unwrap(), + Some("doc-race".to_string()) + ); + + store + .backfill_entity_keys( + "default", + "Doc", + "doc-race", + 2, + KeyIndexBackfillFence { + key_set_signature: repair_signature, + contract_revision: repair_revision, + expected_journal_sequence: 2, + expected_entity_live: true, + expected_snapshot: None, + }, + std::slice::from_ref(&new_path), + ) + .await + .unwrap(); + } +} + +/// Backfill establishes its target contract before replay. A live writer still +/// running the prior contract must advance the revision and fence publication; +/// merely capturing the prior revision would miss this interleaving. +#[tokio::test] +async fn dst_old_contract_live_write_fences_new_contract_backfill() { + for seed in [13] { + let (_guard, _clock, _id) = install_deterministic_context(seed); + let store = SimEventStore::no_faults(seed); + let pid = "default:Doc:doc-contract-race"; + + store + .append_with_index_rows( + pid, + 0, + &[envelope("Create")], + &[key("path", "old-contract")], + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some("v3:old-contract".to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .unwrap(); + store + .mark_key_index_backfilled("default", "Doc", "v3:old-contract") + .await + .unwrap(); + + let repair_revision = store + .begin_key_index_backfill("default", "Doc", "v3:new-contract") + .await + .unwrap(); + assert!( + store + .key_index_backfilled_types("default") + .await + .unwrap() + .is_empty(), + "seed {seed}: beginning repair must withhold old coverage" + ); + + // An old-table actor writes after the new repair target was established. + store + .append_with_index_rows( + pid, + 1, + &[envelope("OldContractWrite")], + &[key("path", "old-contract-after-repair-start")], + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some("v3:old-contract".to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .unwrap(); + assert!( + !store + .mark_key_index_backfilled_if_revision( + "default", + "Doc", + "v3:new-contract", + repair_revision, + ) + .await + .unwrap(), + "seed {seed}: mixed-contract repair must not publish" + ); + assert!( + store + .key_index_backfilled_types("default") + .await + .unwrap() + .is_empty(), + "seed {seed}: failed publication must remain scan-safe" + ); + } +} + +/// A contract fence applies to every repaired row, not only final watermark +/// publication. A writer on another stream can advance the type contract while the +/// repaired entity's journal sequence remains unchanged; the stale pass must not +/// replace that entity's current-contract ownership. +#[tokio::test] +async fn dst_cross_stream_contract_change_fences_repair_rows() { + for seed in [15] { + let (_guard, _clock, _id) = install_deterministic_context(seed); + let store = SimEventStore::no_faults(seed); + let stable_pid = "default:Doc:doc-stable"; + let current_contract = "v3:current-contract"; + let stale_contract = "v3:stale-contract"; + let current_row = key("current_path", "stable-current"); + let stale_row = key("legacy_path", "stable-stale"); + + store + .append_with_index_rows( + stable_pid, + 0, + &[envelope("Create")], + std::slice::from_ref(¤t_row), + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(current_contract.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .unwrap(); + + // The stale pass replays doc-stable at sequence 1 under its target contract. + let stale_revision = store + .begin_key_index_backfill("default", "Doc", stale_contract) + .await + .unwrap(); + + // A different stream moves the type back to the current contract. The + // doc-stable sequence is still 1, so an entity-only sequence fence cannot + // detect this interleaving. + store + .append_with_index_rows( + "default:Doc:doc-live", + 0, + &[envelope("Create")], + &[key("current_path", "live-current")], + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(current_contract.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .unwrap(); + + let stale = store + .backfill_entity_keys( + "default", + "Doc", + "doc-stable", + 1, + KeyIndexBackfillFence { + key_set_signature: stale_contract, + contract_revision: stale_revision, + expected_journal_sequence: 1, + expected_entity_live: true, + expected_snapshot: None, + }, + std::slice::from_ref(&stale_row), + ) + .await; + assert!( + matches!( + stale, + Err(PersistenceError::KeyContractChanged { + expected_revision, + actual_revision, + .. + }) if expected_revision == stale_revision && actual_revision > stale_revision + ), + "seed {seed}: a cross-stream contract change must reject stale repair rows; got {stale:?}" + ); + assert_eq!( + store + .lookup_by_key( + "default", + "Doc", + ¤t_row.key_name, + ¤t_row.key_hash, + ) + .await + .unwrap(), + Some("doc-stable".to_string()), + "seed {seed}: rejected stale repair must preserve current-contract ownership" + ); + assert_eq!( + store + .lookup_by_key("default", "Doc", &stale_row.key_name, &stale_row.key_hash,) + .await + .unwrap(), + None, + "seed {seed}: rejected stale repair must not install obsolete ownership" + ); + } +} + +/// Historical streams can predate declared-key enforcement. If two live streams +/// replay to the same claim, exact repair cannot represent both and must fail closed: +/// preserving the existing owner while preventing the caller from certifying a +/// complete watermark. Silently skipping the second claim would make a partial index +/// look authoritative. +#[tokio::test] +async fn dst_conflicting_backfill_claim_fails_without_partial_mutation() { + for seed in [14] { + let (_guard, _clock, _id) = install_deterministic_context(seed); + let store = SimEventStore::no_faults(seed); + let claimed = key("path", "shared-path"); + + store + .append_with_keys( + "default:Doc:owner", + 0, + &[envelope("Create")], + std::slice::from_ref(&claimed), + ) + .await + .unwrap(); + // A legacy stream written before key co-commit has durable state but no key + // row. Its replay now derives the same current claim as `owner`. + store + .append("default:Doc:legacy-duplicate", 0, &[envelope("Create")]) + .await + .unwrap(); + let repair_signature = "v3:path"; + let repair_revision = store + .begin_key_index_backfill("default", "Doc", repair_signature) + .await + .unwrap(); + + let conflict = store + .backfill_entity_keys( + "default", + "Doc", + "legacy-duplicate", + 1, + KeyIndexBackfillFence { + key_set_signature: repair_signature, + contract_revision: repair_revision, + expected_journal_sequence: 1, + expected_entity_live: true, + expected_snapshot: None, + }, + std::slice::from_ref(&claimed), + ) + .await; + assert!( + matches!(conflict, Err(PersistenceError::Storage(_))), + "seed {seed}: conflicting exact repair must block watermark eligibility" + ); + assert_eq!( + store + .lookup_by_key("default", "Doc", "path", "shared-path") + .await + .unwrap(), + Some("owner".to_string()), + "seed {seed}: failed repair must preserve the established owner" + ); + } +} diff --git a/crates/temper-server/tests/dst_entity_key_reconciliation/seeded_workload.rs b/crates/temper-server/tests/dst_entity_key_reconciliation/seeded_workload.rs new file mode 100644 index 000000000..1bda23752 --- /dev/null +++ b/crates/temper-server/tests/dst_entity_key_reconciliation/seeded_workload.rs @@ -0,0 +1,358 @@ +//! Seed-varied fault, interleaving, and restart workloads. + +use std::collections::BTreeSet; + +use super::*; +use temper_store_sim::{DeterministicRng, SimFaultConfig}; + +const NUM_SEEDS: u64 = 100; + +#[derive(Debug, Default)] +struct FaultCounts { + concurrency: u64, + storage: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ReconciliationTrace { + repair_before_write: bool, + old_contract_writer: bool, + delete_final: bool, + restart_phase: u8, + forced_concurrency_failures: u64, + observed_concurrency_failures: u64, + observed_storage_failures: u64, + replayed_event_types: Vec, + final_owner: Option, +} + +async fn append_exact_with_retry( + store: &SimEventStore, + persistence_id: &str, + expected_sequence: u64, + event_type: &str, + rows: &[EntityKeyRow], + signature: &str, + faults: &mut FaultCounts, +) -> u64 { + let mut event = envelope(event_type); + if event_type == "Delete" { + event.payload = serde_json::json!({ + "action": "Delete", + "from_status": "Ready", + "to_status": "Deleted", + }); + } + for _attempt in 0..128 { + match store + .append_with_index_rows( + persistence_id, + expected_sequence, + std::slice::from_ref(&event), + rows, + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(signature.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + { + Ok(sequence) => return sequence, + Err(PersistenceError::ConcurrencyViolation { .. }) => faults.concurrency += 1, + Err(PersistenceError::Storage(_)) => faults.storage += 1, + Err(error) => panic!("unexpected append failure: {error}"), + } + } + panic!("deterministic append retry budget exhausted for {persistence_id}") +} + +async fn replay_after_restart( + store: &SimEventStore, + persistence_id: &str, + expected_events: usize, +) -> Vec { + // A clone models a fresh store handle after process/actor restart while the + // durable simulated journal remains shared. + let restarted = store.clone(); + let events = restarted + .read_events(persistence_id, 0) + .await + .expect("restart replay must read the durable journal"); + assert_eq!(events.len(), expected_events); + events.into_iter().map(|event| event.event_type).collect() +} + +async fn run_reconciliation_workload(seed: u64) -> ReconciliationTrace { + let (_guard, _clock, _id) = install_deterministic_context(seed); + let mut schedule = DeterministicRng::new(seed ^ 0x00A1_7017_1D57); + let repair_before_write = schedule.next_u64() & 1 == 0; + let old_contract_writer = schedule.next_u64() & 1 == 0; + let delete_final = matches!(schedule.next_u64().checked_rem(3), Some(0)); + let restart_phase = (schedule.next_u64() % 3) as u8; + let forced_concurrency_failures = 1 + schedule.next_u64() % 3; + let store = SimEventStore::new( + seed, + SimFaultConfig { + write_failure_prob: 0.20, + concurrency_violation_prob: 0.12, + read_truncation_prob: 0.0, + snapshot_failure_prob: 0.0, + }, + ); + let persistence_id = "default:Doc:dst-workload"; + let contract_a = "v3|4:path[4:Path]"; + let contract_b = "v3|8:new_path[7:NewPath]"; + let old_row = key("path", "old-path"); + let old_writer_row = key("path", "old-writer-path"); + let final_event_type = if delete_final { "Delete" } else { "LiveWrite" }; + let final_rows = if delete_final { + Vec::new() + } else { + vec![key("new_path", "new-path")] + }; + let old_contract_rows = if delete_final { + Vec::new() + } else { + vec![old_writer_row.clone()] + }; + let mut faults = FaultCounts::default(); + + assert_eq!( + append_exact_with_retry( + &store, + persistence_id, + 0, + "Create", + std::slice::from_ref(&old_row), + contract_a, + &mut faults, + ) + .await, + 1 + ); + store + .mark_key_index_backfilled("default", "Doc", contract_a) + .await + .expect("mark initial coverage"); + if restart_phase == 0 { + replay_after_restart(&store, persistence_id, 1).await; + } + + store.inject_concurrency_violations(persistence_id, forced_concurrency_failures); + if repair_before_write { + let stale_sequence = 1; + let stale_rows = vec![old_row.clone()]; + let first_repair_revision = store + .begin_key_index_backfill("default", "Doc", contract_b) + .await + .expect("begin contract-B repair before live write"); + let (live_rows, live_signature) = if old_contract_writer { + (old_contract_rows.as_slice(), contract_a) + } else { + (final_rows.as_slice(), contract_b) + }; + assert_eq!( + append_exact_with_retry( + &store, + persistence_id, + 1, + final_event_type, + live_rows, + live_signature, + &mut faults, + ) + .await, + 2 + ); + let stale = store + .backfill_entity_keys( + "default", + "Doc", + "dst-workload", + stale_sequence, + KeyIndexBackfillFence { + key_set_signature: contract_b, + contract_revision: first_repair_revision, + expected_journal_sequence: stale_sequence, + expected_entity_live: true, + expected_snapshot: None, + }, + &stale_rows, + ) + .await; + if old_contract_writer { + assert!(matches!( + stale, + Err(PersistenceError::KeyContractChanged { + expected_revision, + actual_revision, + .. + }) if expected_revision == first_repair_revision + && actual_revision > first_repair_revision + )); + assert!( + !store + .mark_key_index_backfilled_if_revision( + "default", + "Doc", + contract_b, + first_repair_revision, + ) + .await + .expect("mixed-contract publication check") + ); + } else { + assert!(matches!( + stale, + Err(PersistenceError::JournalBoundaryChanged { + expected: 1, + actual: 2 + }) + )); + } + } else { + assert_eq!( + append_exact_with_retry( + &store, + persistence_id, + 1, + final_event_type, + &final_rows, + contract_b, + &mut faults, + ) + .await, + 2 + ); + } + + if restart_phase == 1 { + replay_after_restart(&store, persistence_id, 2).await; + } + + let final_revision = store + .begin_key_index_backfill("default", "Doc", contract_b) + .await + .expect("begin converging contract-B repair"); + store + .backfill_entity_keys( + "default", + "Doc", + "dst-workload", + 2, + KeyIndexBackfillFence { + key_set_signature: contract_b, + contract_revision: final_revision, + expected_journal_sequence: 2, + expected_entity_live: !delete_final, + expected_snapshot: None, + }, + &final_rows, + ) + .await + .expect("latest-sequence repair converges"); + assert!( + store + .mark_key_index_backfilled_if_revision("default", "Doc", contract_b, final_revision,) + .await + .expect("publish converged contract-B coverage") + ); + if restart_phase == 2 { + replay_after_restart(&store, persistence_id, 2).await; + } + + let replayed_event_types = replay_after_restart(&store, persistence_id, 2).await; + assert_eq!( + replayed_event_types.last().map(String::as_str), + Some(final_event_type), + "the delete schedule must persist and replay a real terminal boundary" + ); + assert_eq!( + store + .list_entity_ids_by_type("default", "Doc") + .await + .expect("classify final stream liveness") + .contains(&"dst-workload".to_string()), + !delete_final, + "terminal schedules must remain deleted after restart" + ); + let final_owner = store + .lookup_by_key("default", "Doc", "new_path", "new-path") + .await + .expect("read final ownership"); + assert_eq!( + store + .lookup_by_key("default", "Doc", "path", "old-path") + .await + .expect("read released ownership"), + None + ); + assert_eq!( + final_owner, + (!delete_final).then(|| "dst-workload".to_string()) + ); + assert_eq!( + store + .key_index_backfilled_types("default") + .await + .expect("read final watermark"), + vec![("Doc".to_string(), contract_b.to_string())] + ); + + ReconciliationTrace { + repair_before_write, + old_contract_writer, + delete_final, + restart_phase, + forced_concurrency_failures, + observed_concurrency_failures: faults.concurrency, + observed_storage_failures: faults.storage, + replayed_event_types, + final_owner, + } +} + +/// Seeded schedules choose both repair/write orderings, contract races, deletes, +/// restart points, and one-to-three forced concurrency failures while nonzero +/// probabilistic write/concurrency faults exercise retry. Every history converges +/// to replayable contract-B ownership. +#[tokio::test] +async fn dst_seeded_fault_workloads_vary_and_converge_after_restart() { + let mut schedules = BTreeSet::new(); + let mut storage_failures = 0; + for seed in 0..NUM_SEEDS { + let trace = run_reconciliation_workload(seed).await; + assert!( + trace.observed_concurrency_failures >= trace.forced_concurrency_failures, + "seed {seed}: every forced concurrency failure must be observed" + ); + storage_failures += trace.observed_storage_failures; + schedules.insert(( + trace.repair_before_write, + trace.old_contract_writer, + trace.delete_final, + trace.restart_phase, + )); + } + assert!( + schedules.len() >= 8, + "seed must materially vary interleavings, contracts, deletes, and restart points" + ); + assert!( + storage_failures > 0, + "nonzero deterministic write-fault configuration must be exercised" + ); +} + +/// Same seed means the same workload choices, injected failures, durable replay, +/// and final ownership; a neighboring seed must produce a distinct trace. +#[tokio::test] +async fn dst_reconciliation_trace_canary_is_seed_stable() { + let first = run_reconciliation_workload(37).await; + let second = run_reconciliation_workload(37).await; + assert_eq!(first, second); + assert_ne!(first, run_reconciliation_workload(38).await); +} diff --git a/crates/temper-server/tests/dst_platform_random.rs b/crates/temper-server/tests/dst_platform_random.rs index 9a9bead1a..1272b6f47 100644 --- a/crates/temper-server/tests/dst_platform_random.rs +++ b/crates/temper-server/tests/dst_platform_random.rs @@ -169,6 +169,42 @@ async fn dst_random_workload_no_faults() { } } +#[tokio::test] +async fn rejected_first_file_version_action_does_not_publish_a_phantom_entity() { + let seed = 238; + let (_guard, _clock, _id_gen) = install_deterministic_context(seed); + let mut harness = SimPlatformHarness::no_faults(seed); + let tenant = "t-beta"; + let entity_id = "rejected-file-version"; + + harness + .install_app(tenant, "temper-fs") + .await + .expect("temper-fs installs"); + harness.restart().await; + + let _rejected = harness + .dispatch( + tenant, + "FileVersion", + entity_id, + "SetDescription", + serde_json::json!({"description": "unsupported"}), + ) + .await; + + assert!( + harness + .sim_event_store + .dump_journal(&format!("{tenant}:FileVersion:{entity_id}")) + .is_empty(), + "the rejected first action must not append an entity event" + ); + assert_p3_index_store_agreement(&harness) + .await + .expect("the rejected first action must not publish a phantom collection entry"); +} + // ========================================================================= // Test 2: Random workload with event-store faults // ========================================================================= diff --git a/crates/temper-server/tests/e2e_gepa_loop.rs b/crates/temper-server/tests/e2e_gepa_loop.rs index d7f97ed28..45a7ab760 100644 --- a/crates/temper-server/tests/e2e_gepa_loop.rs +++ b/crates/temper-server/tests/e2e_gepa_loop.rs @@ -514,6 +514,46 @@ async fn e2e_gepa_evolution_run_full_lifecycle() { assert_eq!(entity.state.events.len(), 10); } +#[tokio::test] +async fn inline_spec_override_preserves_the_activated_key_contract_epoch() { + let (_guard, _clock, _id_gen) = install_deterministic_context(238); + let harness = SimPlatformHarness::no_faults(238); + harness + .install_app(TENANT, "evolution") + .await + .expect("evolution skill should install"); + + let tenant = temper_runtime::tenant::TenantId::new(TENANT); + let activated_epoch = harness + .platform_state + .server + .registry + .read() + .expect("registry lock") + .get_spec(&tenant, "EvolutionRun") + .expect("EvolutionRun spec") + .table() + .key_contract_activation_epoch; + assert!( + activated_epoch > 0, + "install must activate the key contract" + ); + + harness.register_inline_spec(TENANT, "EvolutionRun", EVOLUTION_RUN_IOA_NO_INTEGRATIONS); + + let retained_epoch = harness + .platform_state + .server + .registry + .read() + .expect("registry lock") + .get_spec(&tenant, "EvolutionRun") + .expect("EvolutionRun spec") + .table() + .key_contract_activation_epoch; + assert_eq!(retained_epoch, activated_epoch); +} + // ========================================================================= // Phase 3: Verification retry loop // ========================================================================= diff --git a/crates/temper-server/tests/ensure_entity_loaded.rs b/crates/temper-server/tests/ensure_entity_loaded.rs index c3c5cd7c4..f050c030e 100644 --- a/crates/temper-server/tests/ensure_entity_loaded.rs +++ b/crates/temper-server/tests/ensure_entity_loaded.rs @@ -1,7 +1,8 @@ use temper_runtime::ActorSystem; use temper_runtime::persistence::{EventMetadata, EventStore, PersistenceEnvelope}; -use temper_runtime::scheduler::sim_now; +use temper_runtime::scheduler::{install_deterministic_context, sim_now}; use temper_runtime::tenant::TenantId; +use temper_store_sim::{SimEventStore, SimFaultConfig}; use temper_store_turso::TursoEventStore; use temper_server::registry::SpecRegistry; @@ -27,6 +28,41 @@ fn build_state_with_turso(system_name: &str, store: TursoEventStore) -> ServerSt state } +fn build_state_with_sim(system_name: &str, store: SimEventStore) -> ServerState { + let mut registry = SpecRegistry::new(); + let csdl = parse_csdl(CSDL_XML).expect("CSDL should parse"); + registry.register_tenant( + "tenant-a", + csdl, + CSDL_XML.to_string(), + &[("Order", ORDER_IOA)], + ); + + let mut state = ServerState::from_registry(ActorSystem::new(system_name), registry); + state.set_storage_stack(StorageStack::from_sim(store, None)); + state +} + +fn snapshot_only_state(entity_id: &str, status: &str) -> Vec { + serde_json::to_vec(&serde_json::json!({ + "entity_type": "Order", + "entity_id": entity_id, + "status": status, + "item_count": 0, + "counters": {}, + "booleans": {}, + "lists": {}, + "fields": {"Id": entity_id, "Status": status}, + "events": [], + "total_event_count": 5, + "events_since_snapshot": 0, + "last_snapshot_sequence_nr": 5, + "sequence_nr": 5, + "processed_idempotency_keys": {} + })) + .unwrap() +} + #[tokio::test] async fn ensure_entity_loaded_returns_false_when_no_transition_table_exists() { let db_path = @@ -104,6 +140,118 @@ async fn ensure_entity_loaded_returns_true_for_indexed_entity_without_persistenc ); } +#[tokio::test] +async fn ensure_entity_loaded_hydrates_cold_snapshot_only_entity() { + let db_path = std::env::temp_dir().join(format!( + "temper-ensure-snapshot-only-{}.db", + uuid::Uuid::new_v4() + )); + let store = TursoEventStore::new(&format!("file:{}", db_path.display()), None) + .await + .unwrap(); + let tenant = TenantId::new("tenant-a"); + let entity_id = "ord-snapshot-only"; + store + .save_snapshot( + &format!("{tenant}:Order:{entity_id}"), + 5, + &snapshot_only_state(entity_id, "Draft"), + ) + .await + .unwrap(); + let state = build_state_with_turso("test-ensure-snapshot-only", store); + + assert!( + state + .ensure_entity_loaded(&tenant, "Order", entity_id) + .await + ); + assert!(state.entity_exists(&tenant, "Order", entity_id)); + let hydrated = state + .get_tenant_entity_state(&tenant, "Order", entity_id) + .await + .unwrap(); + assert_eq!( + hydrated.state.sequence_nr, 0, + "snapshot-only state starts a new journal generation at sequence zero" + ); + assert_eq!( + hydrated.state.total_event_count, 5, + "snapshot logical history remains intact while journal coordinates reset" + ); + assert_eq!(hydrated.state.status, "Draft"); +} + +#[tokio::test] +async fn ensure_entity_loaded_rejects_deleted_snapshot_only_entity() { + let db_path = std::env::temp_dir().join(format!( + "temper-ensure-deleted-snapshot-only-{}.db", + uuid::Uuid::new_v4() + )); + let store = TursoEventStore::new(&format!("file:{}", db_path.display()), None) + .await + .unwrap(); + let tenant = TenantId::new("tenant-a"); + let entity_id = "ord-deleted-snapshot-only"; + store + .save_snapshot( + &format!("{tenant}:Order:{entity_id}"), + 5, + &snapshot_only_state(entity_id, "Deleted"), + ) + .await + .unwrap(); + let state = build_state_with_turso("test-ensure-deleted-snapshot-only", store); + + assert!( + !state + .ensure_entity_loaded(&tenant, "Order", entity_id) + .await + ); + assert!(!state.entity_exists(&tenant, "Order", entity_id)); +} + +#[tokio::test] +async fn ensure_entity_loaded_rehydrates_same_sequence_snapshot_replacement() { + let db_path = std::env::temp_dir().join(format!( + "temper-ensure-replaced-snapshot-{}.db", + uuid::Uuid::new_v4() + )); + let store = TursoEventStore::new(&format!("file:{}", db_path.display()), None) + .await + .unwrap(); + let tenant = TenantId::new("tenant-a"); + let entity_id = "ord-replaced-snapshot"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + store + .save_snapshot(&persistence_id, 5, &snapshot_only_state(entity_id, "Draft")) + .await + .unwrap(); + let state = build_state_with_turso("test-ensure-replaced-snapshot", store.clone()); + assert!( + state + .ensure_entity_loaded(&tenant, "Order", entity_id) + .await + ); + + store + .save_snapshot( + &persistence_id, + 5, + &snapshot_only_state(entity_id, "Deleted"), + ) + .await + .unwrap(); + + assert!( + !state + .ensure_entity_loaded(&tenant, "Order", entity_id) + .await, + "a resident actor hydrated from the replaced live snapshot must not hide the Deleted generation" + ); + assert!(!state.entity_exists(&tenant, "Order", entity_id)); +} + #[tokio::test] async fn list_entity_ids_lazy_populates_only_requested_type() { let db_path = std::env::temp_dir().join(format!( @@ -233,21 +381,13 @@ async fn delete_writes_tombstone_and_deleted_entity_stays_out_of_list_after_rest #[tokio::test] async fn delete_failure_does_not_remove_live_entity_from_index() { - let db_path = std::env::temp_dir().join(format!( - "temper-delete-tombstone-failure-{}.db", - uuid::Uuid::new_v4() - )); - let db_url = format!("file:{}", db_path.display()); - let store = TursoEventStore::new(&db_url, None) - .await - .expect("create local turso db"); + let (_guard, _clock, _ids) = install_deterministic_context(909); + let store = SimEventStore::no_faults(909); let tenant = TenantId::new("tenant-a"); let entity_type = "Order"; let entity_id = "ord-delete-failure"; - let persistence_id = format!("{tenant}:{entity_type}:{entity_id}"); - - let state = build_state_with_turso("test-delete-tombstone-failure", store.clone()); + let state = build_state_with_sim("test-delete-tombstone-failure", store.clone()); state .get_or_create_tenant_entity( &tenant, @@ -258,31 +398,10 @@ async fn delete_failure_does_not_remove_live_entity_from_index() { .await .expect("create entity"); - // Force delete persistence to fail by appending an external event first, - // so the actor's expected sequence is stale. - let expected_sequence = store - .read_events(&persistence_id, 0) - .await - .expect("read seeded events") - .last() - .map(|event| event.sequence_nr) - .expect("created event sequence present"); - let external = PersistenceEnvelope { - sequence_nr: 0, - event_type: "ExternalWrite".to_string(), - payload: serde_json::json!({"action": "ExternalWrite"}), - metadata: EventMetadata { - event_id: uuid::Uuid::new_v4(), - causation_id: uuid::Uuid::new_v4(), - correlation_id: uuid::Uuid::new_v4(), - timestamp: sim_now(), - actor_id: "concurrency-racer".to_string(), - }, - }; - store - .append(&persistence_id, expected_sequence, &[external]) - .await - .expect("append external race event"); + store.restore_faults(SimFaultConfig { + write_failure_prob: 1.0, + ..SimFaultConfig::none() + }); let response = state .delete_tenant_entity(&tenant, entity_type, entity_id) @@ -313,8 +432,6 @@ async fn delete_failure_does_not_remove_live_entity_from_index() { live.state.status, "Deleted", "failed delete must not advance state to Deleted" ); - - let _ = std::fs::remove_file(db_path); } #[tokio::test] diff --git a/crates/temper-server/tests/file_value_fast_path.rs b/crates/temper-server/tests/file_value_fast_path.rs index 165b29ed6..92de73637 100644 --- a/crates/temper-server/tests/file_value_fast_path.rs +++ b/crates/temper-server/tests/file_value_fast_path.rs @@ -4,6 +4,7 @@ use sha2::{Digest, Sha256}; use temper_runtime::ActorSystem; use temper_runtime::persistence::EventStore; use temper_runtime::tenant::TenantId; +use temper_server::key_index::canonical_key_hash; use temper_server::registry::SpecRegistry; use temper_server::registry::{EntityLevelSummary, EntityVerificationResult, VerificationStatus}; use temper_server::request_context::AgentContext; @@ -11,6 +12,7 @@ use temper_server::state::IndexedFileStreamRead; use temper_server::storage::StorageStack; use temper_server::{ServerState, build_router}; use temper_spec::csdl::parse_csdl; +use temper_store_sim::SimEventStore; use temper_store_turso::TursoEventStore; use tower::ServiceExt; @@ -106,6 +108,10 @@ name = "version_count" type = "counter" initial = "0" +[[key]] +name = "workspace_path" +properties = ["workspace_id", "path"] + [[action]] name = "Create" kind = "input" @@ -129,6 +135,17 @@ effect = [ ] "#; +fn file_path_key_hash(workspace_id: &str, path: &str) -> String { + canonical_key_hash( + "workspace_path", + &["workspace_id".to_string(), "path".to_string()], + serde_json::json!({"workspace_id": workspace_id, "path": path}) + .as_object() + .expect("file key fields"), + ) + .expect("complete File declared key") +} + const FILE_IOA: &str = r#" [automaton] name = "File" @@ -160,6 +177,10 @@ name = "version_count" type = "counter" initial = "0" +[[key]] +name = "workspace_path" +properties = ["workspace_id", "path"] + [[action]] name = "Create" kind = "input" @@ -218,6 +239,21 @@ async fn build_turso_file_state(test_name: &str) -> (ServerState, TursoEventStor (state, store) } +fn build_sim_file_state(test_name: &str) -> (ServerState, SimEventStore) { + let store = SimEventStore::no_faults(238); + let mut registry = SpecRegistry::new(); + let csdl = parse_csdl(FILE_CSDL_XML).expect("file CSDL should parse"); + registry.register_tenant( + "default", + csdl, + FILE_CSDL_XML.to_string(), + &[("File", FILE_IOA)], + ); + let mut state = ServerState::from_registry(ActorSystem::new(test_name), registry); + state.set_storage_stack(StorageStack::from_sim(store.clone(), None)); + (state, store) +} + fn mark_file_verified(state: &ServerState) { let mut registry = state.registry.write().unwrap(); registry.set_verification_status( @@ -303,6 +339,75 @@ async fn create_file_with_initial_stream_content_projects_only_ready_content() { assert_local_blob(data_dir.path(), &expected_hash, body).await; } +#[tokio::test] +async fn create_file_with_initial_stream_content_co_commits_declared_key() { + let (mut state, store) = build_sim_file_state("atomic-initial-content-key"); + let tenant = TenantId::default(); + let data_dir = tempfile::tempdir().expect("temp data dir"); + state.data_dir = data_dir.path().to_path_buf(); + let body = b"keyed atomic File value"; + + state + .create_file_with_initial_stream_content( + &tenant, + "fl-atomic-keyed", + serde_json::json!({ + "name": "atomic.md", + "path": "/atomic.md", + "directory_id": "dir-root", + "workspace_id": "ws-root", + "mime_type": "text/markdown", + }), + body, + "text/markdown", + &AgentContext::for_service("test-writer"), + ) + .await + .expect("keyed initial-content create"); + assert_eq!( + store + .lookup_by_key( + "default", + "File", + "workspace_path", + &file_path_key_hash("ws-root", "/atomic.md"), + ) + .await + .expect("initial-content key lookup"), + Some("fl-atomic-keyed".to_string()), + "the synthetic initial-content path must co-commit declared-key ownership" + ); + + let duplicate = state + .create_file_with_initial_stream_content( + &tenant, + "fl-atomic-duplicate", + serde_json::json!({ + "name": "duplicate.md", + "path": "/atomic.md", + "directory_id": "dir-root", + "workspace_id": "ws-root", + "mime_type": "text/markdown", + }), + b"duplicate bytes", + "text/markdown", + &AgentContext::for_service("test-writer"), + ) + .await; + assert!( + duplicate.is_err(), + "a duplicate File declared key must be rejected" + ); + assert!( + store + .read_events("default:File:fl-atomic-duplicate", 0) + .await + .expect("duplicate File journal read") + .is_empty(), + "a rejected synthetic create must leave no durable File events" + ); +} + #[tokio::test] async fn put_file_stream_content_writes_native_blob_and_dispatches_update() { let (mut state, _store) = build_turso_file_state("native-write").await; diff --git a/crates/temper-server/tests/key_authority_catalog_fast_read.rs b/crates/temper-server/tests/key_authority_catalog_fast_read.rs new file mode 100644 index 000000000..6c0a47730 --- /dev/null +++ b/crates/temper-server/tests/key_authority_catalog_fast_read.rs @@ -0,0 +1,220 @@ +//! Regression for the catalog fast-read flag crossing an authoritative key boundary. + +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use temper_runtime::ActorSystem; +use temper_runtime::persistence::{ + EventMetadata, EventStore, PersistenceEnvelope, PersistenceError, +}; +use temper_runtime::scheduler::{install_deterministic_context, sim_now, sim_uuid}; +use temper_runtime::tenant::TenantId; +use temper_server::entity_actor::EntityEvent; +use temper_server::registry::SpecRegistry; +use temper_server::storage::{ + BackendLabel, BoxedEventStore, EntityCatalogRow, QueryPlaneStore, QueryProjectionFieldsRow, + StorageStack, +}; +use temper_server::{ServerState, build_router}; +use temper_spec::csdl::parse_csdl; +use temper_store_sim::SimEventStore; +use tower::ServiceExt; + +const CSDL_XML: &str = include_str!("../../../test-fixtures/specs/model.csdl.xml"); +const ORDER_IOA: &str = include_str!("../../../test-fixtures/specs/order.ioa.toml"); + +#[derive(Default)] +struct StaleCatalog { + rows: Mutex>, +} + +#[async_trait] +impl QueryPlaneStore for StaleCatalog { + async fn upsert_projection( + &self, + _tenant: &str, + _entity_type: &str, + entity_id: &str, + status: &str, + fields: &serde_json::Value, + state: &serde_json::Value, + sequence_nr: u64, + ) -> Result<(), PersistenceError> { + self.rows.lock().expect("catalog lock").insert( + entity_id.to_string(), + EntityCatalogRow { + entity_id: entity_id.to_string(), + status: status.to_string(), + fields: fields.clone(), + state: Some(state.clone()), + sequence_nr, + }, + ); + Ok(()) + } + + async fn remove_projection( + &self, + _tenant: &str, + _entity_type: &str, + entity_id: &str, + ) -> Result<(), PersistenceError> { + self.rows.lock().expect("catalog lock").remove(entity_id); + Ok(()) + } + + async fn query_field_index( + &self, + _tenant: &str, + _entity_type: &str, + _where_clause: &str, + _params: Vec, + ) -> Result>, PersistenceError> { + Ok(None) + } + + async fn load_projection_fields_many( + &self, + _tenant: &str, + _entity_type: &str, + _entity_ids: &[String], + _field_names: &[&str], + ) -> Result>, PersistenceError> { + Ok(None) + } + + async fn load_entity_catalog_rows( + &self, + _tenant: &str, + _entity_type: &str, + entity_ids: &[String], + ) -> Result>, PersistenceError> { + let rows = self.rows.lock().expect("catalog lock"); + Ok(Some( + entity_ids + .iter() + .filter_map(|id| rows.get(id).cloned()) + .collect(), + )) + } + + async fn projected_entity_counts_by_tenant( + &self, + ) -> Result>, PersistenceError> { + Ok(None) + } +} + +fn state_with_store(name: &str, events: &SimEventStore, catalog: Arc) -> ServerState { + let csdl = parse_csdl(CSDL_XML).expect("CSDL parse"); + let mut registry = SpecRegistry::new(); + registry.register_tenant( + "default", + csdl, + CSDL_XML.to_string(), + &[("Order", ORDER_IOA)], + ); + let mut state = ServerState::from_registry(ActorSystem::new(name), registry); + state.set_storage_stack(StorageStack::new( + BackendLabel::Sim, + BoxedEventStore::new(events.clone()), + None, + None, + None, + None, + Some(catalog), + None, + None, + None, + )); + state +} + +/// This integration-test binary contains one test, so setting the process-local +/// flag cannot leak into another test. An authoritative keyed candidate must still +/// materialize from its journal/actor when the catalog is stale. +#[tokio::test] +async fn catalog_fast_read_cannot_override_authoritative_key_materialization() { + unsafe { + std::env::set_var("TEMPER_ODATA_CATALOG_FAST_READ", "true"); + } + let (_guard, _clock, _ids) = install_deterministic_context(240); + let events = SimEventStore::no_faults(240); + let catalog = Arc::new(StaleCatalog::default()); + let tenant = TenantId::default(); + let entity_id = "legacy-deleted"; + let state = state_with_store("arn238-fast-read-seed", &events, catalog.clone()); + + state + .get_or_create_tenant_entity( + &tenant, + "Order", + entity_id, + serde_json::json!({"WorkspaceId": "ws", "Path": "/stale"}), + ) + .await + .expect("create keyed entity"); + state.populate_key_index_from_snapshots(&tenant).await; + + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let sequence_nr = events + .read_events(&persistence_id, 0) + .await + .expect("read history") + .last() + .expect("created event") + .sequence_nr; + let timestamp = sim_now(); + let tombstone = EntityEvent { + action: "Deleted".to_string(), + from_status: "Draft".to_string(), + to_status: "Deleted".to_string(), + timestamp, + params: serde_json::json!({}), + idempotency_key: None, + }; + events + .append( + &persistence_id, + sequence_nr, + &[PersistenceEnvelope { + sequence_nr: sequence_nr + 1, + event_type: "Deleted".to_string(), + payload: serde_json::to_value(tombstone).expect("serialize tombstone"), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: persistence_id.clone(), + }, + }], + ) + .await + .expect("append legacy event-only tombstone"); + drop(state); + + let restarted = state_with_store("arn238-fast-read-restart", &events, catalog); + let response = build_router(restarted) + .oneshot( + Request::builder() + .uri("/tdata/Orders?$filter=WorkspaceId%20eq%20%27ws%27%20and%20Path%20eq%20%27%2Fstale%27") + .body(Body::empty()) + .expect("request"), + ) + .await + .expect("response"); + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), 1_000_000) + .await + .expect("response body"); + let json: serde_json::Value = serde_json::from_slice(&body).expect("OData JSON"); + assert_eq!( + json["value"], + serde_json::json!([]), + "the durable tombstone must win even when catalog fast-read is enabled" + ); +} diff --git a/crates/temper-server/tests/key_ownership_postgres_repair.rs b/crates/temper-server/tests/key_ownership_postgres_repair.rs new file mode 100644 index 000000000..01c1c9925 --- /dev/null +++ b/crates/temper-server/tests/key_ownership_postgres_repair.rs @@ -0,0 +1,468 @@ +//! Real-Postgres regression for pre-v3 deleted and orphaned key owners. + +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use temper_runtime::ActorSystem; +use temper_runtime::persistence::{EventMetadata, EventStore, PersistenceEnvelope}; +use temper_runtime::scheduler::{sim_now, sim_uuid}; +use temper_runtime::tenant::TenantId; +use temper_server::entity_actor::EntityEvent; +use temper_server::key_index::{canonical_key_hash, declared_key_set_signature}; +use temper_server::registry::SpecRegistry; +use temper_server::{ServerState, StorageStack, build_router}; +use temper_spec::csdl::parse_csdl; +use temper_store_postgres::PostgresEventStore; +use tower::ServiceExt; + +const CSDL_XML: &str = r#" + + + + + + + + + + + + + + + +"#; +const DOC_IOA: &str = include_str!("../../../test-fixtures/specs/keyed_doc.ioa.toml"); +const DATA_ONLY_DOC_IOA: &str = r#" +[automaton] +name = "Doc" +states = ["Ready"] +initial = "Ready" + +[[state]] +name = "WorkspaceId" +type = "string" +initial = "" + +[[state]] +name = "Path" +type = "string" +initial = "" + +[[key]] +name = "path" +properties = ["WorkspaceId", "Path"] +"#; + +fn key_hash(workspace: &str, path: &str) -> String { + canonical_key_hash( + "path", + &["WorkspaceId".to_string(), "Path".to_string()], + serde_json::json!({"WorkspaceId": workspace, "Path": path}) + .as_object() + .expect("key fields"), + ) + .expect("complete key") +} + +fn server_with_postgres_spec( + tenant: &TenantId, + store: PostgresEventStore, + ioa: &'static str, +) -> ServerState { + let csdl = parse_csdl(CSDL_XML).expect("CSDL parse"); + let mut registry = SpecRegistry::new(); + registry.register_tenant(tenant.as_str(), csdl, CSDL_XML.to_string(), &[("Doc", ioa)]); + let mut state = ServerState::from_registry(ActorSystem::new("arn238-pg-repair"), registry); + state.set_storage_stack(StorageStack::from_postgres(store)); + state +} + +fn server_with_postgres(tenant: &TenantId, store: PostgresEventStore) -> ServerState { + server_with_postgres_spec(tenant, store, DOC_IOA) +} + +async fn read_migrated_doc_by_key( + state: &ServerState, + tenant: &TenantId, +) -> (StatusCode, serde_json::Value) { + let response = build_router(state.clone()) + .oneshot( + Request::builder() + .uri( + "/tdata/Docs?$filter=WorkspaceId%20eq%20%27ws%27%20and%20Path%20eq%20%27%2Fmigrated%27", + ) + .header("x-tenant-id", tenant.as_str()) + .body(Body::empty()) + .expect("catalog-only keyed request"), + ) + .await + .expect("catalog-only keyed response"); + let status = response.status(); + let bytes = axum::body::to_bytes(response.into_body(), 1_000_000) + .await + .expect("catalog-only keyed response bytes"); + let body = serde_json::from_slice(&bytes).expect("catalog-only keyed response JSON"); + (status, body) +} + +/// A type cannot receive a complete watermark while stale rows owned by a deleted +/// journal stream or a key-index-only phantom remain outside the repair enumeration. +#[tokio::test] +async fn postgres_backfill_purges_deleted_and_orphaned_key_owners_before_watermark() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => return, + }; + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect to Postgres"); + temper_store_postgres::migration::run_migrations(&pool) + .await + .expect("run Postgres migrations"); + let store = PostgresEventStore::new(pool.clone()); + let tenant = TenantId::new(format!("arn238-key-repair-{}", sim_uuid())); + let state = server_with_postgres(&tenant, store.clone()); + let deleted_id = "legacy-deleted"; + let deleted_hash = key_hash("ws", "/reclaim"); + + state + .get_or_create_tenant_entity( + &tenant, + "Doc", + deleted_id, + serde_json::json!({"WorkspaceId": "ws", "Path": "/reclaim"}), + ) + .await + .expect("create legacy key owner"); + let persistence_id = format!("{tenant}:Doc:{deleted_id}"); + let sequence_nr = store + .read_events(&persistence_id, 0) + .await + .expect("read legacy history") + .last() + .expect("created event") + .sequence_nr; + let timestamp = sim_now(); + let deleted = EntityEvent { + action: "Deleted".to_string(), + from_status: "New".to_string(), + to_status: "Deleted".to_string(), + timestamp, + params: serde_json::json!({}), + idempotency_key: None, + }; + store + .append( + &persistence_id, + sequence_nr, + &[PersistenceEnvelope { + sequence_nr: sequence_nr + 1, + event_type: "Deleted".to_string(), + payload: serde_json::to_value(deleted).expect("serialize tombstone"), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: persistence_id.clone(), + }, + }], + ) + .await + .expect("append pre-v3 event-only tombstone"); + let stale_snapshot = serde_json::json!({ + "entity_type": "Doc", + "entity_id": deleted_id, + "status": "New", + "item_count": 0, + "total_event_count": sequence_nr + 2, + "fields": {"WorkspaceId": "ws", "Path": "/reclaim"}, + }); + store + .save_snapshot( + &persistence_id, + sequence_nr + 2, + &serde_json::to_vec(&stale_snapshot).expect("serialize stale snapshot"), + ) + .await + .expect("seed newer stale live snapshot"); + store + .upsert_query_projection( + tenant.as_str(), + "Doc", + deleted_id, + "New", + &serde_json::json!({"WorkspaceId": "ws", "Path": "/reclaim"}), + sequence_nr + 3, + ) + .await + .expect("seed newest stale live catalog row"); + + let orphan_hash = key_hash("ws", "/orphan"); + sqlx::query( + "INSERT INTO entity_key_index \ + (tenant, entity_type, key_name, key_hash, entity_id, sequence_nr) \ + VALUES ($1, 'Doc', 'path', $2, 'orphan-owner', 0)", + ) + .bind(tenant.as_str()) + .bind(&orphan_hash) + .execute(&pool) + .await + .expect("seed key-index-only orphan"); + + for (hash, owner) in [(&deleted_hash, deleted_id), (&orphan_hash, "orphan-owner")] { + assert_eq!( + store + .lookup_by_key(tenant.as_str(), "Doc", "path", hash) + .await + .expect("legacy key lookup"), + Some(owner.to_string()), + "precondition: stale owner is present before repair" + ); + } + + state.populate_key_index_from_snapshots(&tenant).await; + + for hash in [&deleted_hash, &orphan_hash] { + assert_eq!( + store + .lookup_by_key(tenant.as_str(), "Doc", "path", hash) + .await + .expect("repaired key lookup"), + None, + "repair must purge deleted and key-index-only owners" + ); + } + let table = TransitionTableForTest::doc(); + let signature = declared_key_set_signature(&table.keys); + assert!( + store + .key_index_backfilled_types(tenant.as_str()) + .await + .expect("watermarks") + .contains(&("Doc".to_string(), signature)), + "the watermark is valid only after every stale owner is purged" + ); + + let reclaimed = state + .get_or_create_tenant_entity( + &tenant, + "Doc", + "replacement", + serde_json::json!({"WorkspaceId": "ws", "Path": "/reclaim"}), + ) + .await + .expect("reclaim formerly stale key"); + assert!(reclaimed.success); + assert_eq!( + store + .lookup_by_key(tenant.as_str(), "Doc", "path", &deleted_hash) + .await + .expect("reclaimed lookup"), + Some("replacement".to_string()) + ); +} + +/// Migrated deployments can have a durable query-plane catalog row without a +/// journal: ADR-0077 populated the catalog directly from snapshots rather than +/// replaying historical events. Key reconciliation must include that compatibility +/// shape before publishing authoritative coverage, or a keyed miss can hide the +/// existing owner and admit a duplicate claim after restart. +#[tokio::test] +async fn postgres_backfill_reconstructs_catalog_only_owner_before_watermark() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => return, + }; + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect to Postgres"); + temper_store_postgres::migration::run_migrations(&pool) + .await + .expect("run Postgres migrations"); + let store = PostgresEventStore::new(pool); + let tenant = TenantId::new(format!("arn238-catalog-only-pg-{}", sim_uuid())); + let state = server_with_postgres(&tenant, store.clone()); + let owner_id = "migrated-owner"; + let hash = key_hash("ws", "/migrated"); + let fields = serde_json::json!({"WorkspaceId": "ws", "Path": "/migrated"}); + + store + .upsert_query_projection(tenant.as_str(), "Doc", owner_id, "New", &fields, 7) + .await + .expect("seed migrated catalog-only entity"); + assert!( + store + .read_events(&format!("{tenant}:Doc:{owner_id}"), 0) + .await + .expect("catalog-only journal read") + .is_empty(), + "precondition: migrated owner has no journal" + ); + + let (pre_repair_status, pre_repair_body) = read_migrated_doc_by_key(&state, &tenant).await; + assert_eq!(pre_repair_status, StatusCode::OK); + assert_eq!( + pre_repair_body["value"][0]["entity_id"], owner_id, + "scan-safe reads must materialize catalog-only migration owners before coverage" + ); + + state.populate_key_index_from_snapshots(&tenant).await; + + assert_eq!( + store + .lookup_by_key(tenant.as_str(), "Doc", "path", &hash) + .await + .expect("reconstructed key lookup"), + Some(owner_id.to_string()), + "repair must reconstruct the durable catalog-only owner before coverage" + ); + let table = TransitionTableForTest::doc(); + let signature = declared_key_set_signature(&table.keys); + assert!( + store + .key_index_backfilled_types(tenant.as_str()) + .await + .expect("watermarks") + .contains(&("Doc".to_string(), signature)), + "coverage may publish only after the catalog-only owner is indexed" + ); + + let (post_repair_status, post_repair_body) = read_migrated_doc_by_key(&state, &tenant).await; + assert_eq!(post_repair_status, StatusCode::OK); + assert_eq!( + post_repair_body["value"][0]["entity_id"], owner_id, + "the fenced key hit must materialize the exact catalog generation" + ); + + let duplicate = state + .get_or_create_tenant_entity( + &tenant, + "Doc", + "duplicate", + serde_json::json!({"WorkspaceId": "ws", "Path": "/migrated"}), + ) + .await; + assert!( + duplicate.is_err(), + "the reconstructed durable owner must reject a duplicate claim" + ); +} + +/// A legacy field-index row proves that an entity exists but does not contain a +/// complete state object from which every declared key component can be derived. +/// Repair must keep reads scan-safe by withholding coverage instead of certifying +/// that the partial row owns no key. +#[tokio::test] +async fn postgres_backfill_withholds_watermark_for_field_index_only_entity() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => return, + }; + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect to Postgres"); + temper_store_postgres::migration::run_migrations(&pool) + .await + .expect("run Postgres migrations"); + let store = PostgresEventStore::new(pool.clone()); + let tenant = TenantId::new(format!("arn238-field-only-pg-{}", sim_uuid())); + let state = server_with_postgres(&tenant, store.clone()); + + sqlx::query( + "INSERT INTO entity_field_index \ + (tenant, entity_type, entity_id, field_name, field_value, status) \ + VALUES ($1, 'Doc', 'partial-owner', 'WorkspaceId', 'ws', 'New')", + ) + .bind(tenant.as_str()) + .execute(&pool) + .await + .expect("seed field-index-only compatibility row"); + + state.populate_key_index_from_snapshots(&tenant).await; + + let table = TransitionTableForTest::doc(); + let signature = declared_key_set_signature(&table.keys); + assert!( + !store + .key_index_backfilled_types(tenant.as_str()) + .await + .expect("watermarks") + .contains(&("Doc".to_string(), signature)), + "a field-index-only entity cannot be reconstructed and must block coverage" + ); +} + +/// The native PostgreSQL data-only optimization must claim a declared key in the +/// same transaction as its first event and projection. A conflicting claim rolls +/// all three writes back. +#[tokio::test] +async fn postgres_native_data_only_create_co_commits_declared_keys() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => return, + }; + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect to Postgres"); + temper_store_postgres::migration::run_migrations(&pool) + .await + .expect("run Postgres migrations"); + let store = PostgresEventStore::new(pool); + let tenant = TenantId::new(format!("arn238-data-only-pg-{}", sim_uuid())); + let state = server_with_postgres_spec(&tenant, store.clone(), DATA_ONLY_DOC_IOA); + let hash = key_hash("ws", "/native"); + + let created = state + .try_create_data_only_tenant_entity( + &tenant, + "Doc", + "native-a", + serde_json::json!({"WorkspaceId": "ws", "Path": "/native"}), + ) + .await + .expect("native data-only create result") + .expect("eligible native data-only create"); + assert!(created.success); + assert_eq!( + store + .lookup_by_key(tenant.as_str(), "Doc", "path", &hash) + .await + .expect("native key lookup"), + Some("native-a".to_string()) + ); + + let duplicate = state + .try_create_data_only_tenant_entity( + &tenant, + "Doc", + "native-b", + serde_json::json!({"WorkspaceId": "ws", "Path": "/native"}), + ) + .await; + assert!(duplicate.is_err(), "a duplicate native claim must fail"); + assert!( + store + .read_events(&format!("{tenant}:Doc:native-b"), 0) + .await + .expect("duplicate journal read") + .is_empty(), + "the conflicting native transaction must not insert an event" + ); + assert_eq!( + store + .lookup_by_key(tenant.as_str(), "Doc", "path", &hash) + .await + .expect("owner after duplicate"), + Some("native-a".to_string()), + "the rejected transaction must preserve the original owner" + ); +} + +struct TransitionTableForTest; + +impl TransitionTableForTest { + fn doc() -> temper_jit::table::TransitionTable { + temper_jit::table::TransitionTable::from_ioa_source(DOC_IOA) + } +} diff --git a/crates/temper-server/tests/key_ownership_write_surfaces.rs b/crates/temper-server/tests/key_ownership_write_surfaces.rs new file mode 100644 index 000000000..2cdd85c9c --- /dev/null +++ b/crates/temper-server/tests/key_ownership_write_surfaces.rs @@ -0,0 +1,529 @@ +//! Regressions for declared-key ownership across non-action write surfaces. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::{Arc, RwLock}; +use std::time::Duration; + +use async_trait::async_trait; +use temper_jit::table::TransitionTable; +use temper_jit::table::types::DeclaredKey; +use temper_runtime::ActorSystem; +use temper_runtime::persistence::{EventStore, PersistenceError}; +use temper_runtime::scheduler::{install_deterministic_context, sim_uuid}; +use temper_runtime::tenant::TenantId; +use temper_server::entity_actor::EntityEvent; +use temper_server::key_index::{canonical_key_hash, declared_key_set_signature}; +use temper_server::registry::SpecRegistry; +use temper_server::storage::{ + BackendLabel, BoxedEventStore, QueryPlaneStore, QueryProjectionFieldsRow, StorageStack, +}; +use temper_server::{EntityActor, EntityMsg, EntityResponse, ServerState}; +use temper_spec::csdl::parse_csdl; +use temper_store_sim::SimEventStore; + +#[path = "key_ownership_write_surfaces/audit_only_repair.rs"] +mod audit_only_repair; +#[path = "key_ownership_write_surfaces/field_update_recovery.rs"] +mod field_update_recovery; +#[path = "key_ownership_write_surfaces/key_contract_aba.rs"] +mod key_contract_aba; +#[path = "key_ownership_write_surfaces/tombstone_snapshot_recovery.rs"] +mod tombstone_snapshot_recovery; + +const CSDL_XML: &str = include_str!("../../../test-fixtures/specs/model.csdl.xml"); +const DOC_IOA: &str = include_str!("../../../test-fixtures/specs/keyed_doc.ioa.toml"); +const DATA_ONLY_DOC_IOA: &str = r#" +[automaton] +name = "Doc" +states = ["Ready"] +initial = "Ready" + +[[state]] +name = "WorkspaceId" +type = "string" +initial = "" + +[[state]] +name = "Path" +type = "string" +initial = "" + +[[key]] +name = "path" +properties = ["WorkspaceId", "Path"] +"#; +fn doc_key_hash(workspace: &str, path: &str) -> String { + canonical_key_hash( + "path", + &["WorkspaceId".to_string(), "Path".to_string()], + serde_json::json!({"WorkspaceId": workspace, "Path": path}) + .as_object() + .expect("key fields"), + ) + .expect("complete declared key") +} + +async fn action( + actor: &temper_runtime::actor::ActorRef, + name: &str, + params: serde_json::Value, +) -> EntityResponse { + actor + .ask( + EntityMsg::Action { + name: name.to_string(), + params, + cross_entity_booleans: BTreeMap::new(), + idempotency_key: None, + }, + Duration::from_secs(5), + ) + .await + .expect("action response") +} + +async fn update( + actor: &temper_runtime::actor::ActorRef, + fields: serde_json::Value, + replace: bool, +) -> EntityResponse { + update_with_idempotency( + actor, + fields, + replace, + format!("test-field-update:{}", sim_uuid()), + ) + .await +} + +async fn update_with_idempotency( + actor: &temper_runtime::actor::ActorRef, + fields: serde_json::Value, + replace: bool, + idempotency_key: String, +) -> EntityResponse { + actor + .ask( + EntityMsg::UpdateFields { + fields, + replace, + idempotency_key, + }, + Duration::from_secs(5), + ) + .await + .expect("field update response") +} + +async fn state(actor: &temper_runtime::actor::ActorRef) -> EntityResponse { + actor + .ask(EntityMsg::GetState, Duration::from_secs(5)) + .await + .expect("state response") +} + +/// PATCH and PUT are durable writes: they must atomically move declared-key +/// ownership, roll back a uniqueness reject, and replay after actor restart. +#[tokio::test] +async fn field_updates_reconcile_keys_and_survive_restart() { + let (_guard, _clock, _ids) = install_deterministic_context(238); + let sim = SimEventStore::no_faults(238); + let events = BoxedEventStore::new(sim.clone()); + let table = Arc::new(RwLock::new(TransitionTable::from_ioa_source(DOC_IOA))); + let system = ActorSystem::new("arn238-field-updates"); + + let first = system.spawn( + EntityActor::with_persistence( + "Doc", + "doc-a", + table.clone(), + serde_json::json!({}), + events.clone(), + BackendLabel::Sim, + ) + .with_tenant("default"), + "doc-a", + ); + let blocker = system.spawn( + EntityActor::with_persistence( + "Doc", + "doc-b", + table.clone(), + serde_json::json!({}), + events.clone(), + BackendLabel::Sim, + ) + .with_tenant("default"), + "doc-b", + ); + + assert!( + action( + &first, + "Create", + serde_json::json!({"WorkspaceId": "ws", "Path": "/old"}), + ) + .await + .success + ); + assert!( + action( + &blocker, + "Create", + serde_json::json!({"WorkspaceId": "ws", "Path": "/blocked"}), + ) + .await + .success + ); + let created_sequence = state(&first).await.state.sequence_nr; + + let patched = update( + &first, + serde_json::json!({"Path": "/patched", "Note": "keep-until-put"}), + false, + ) + .await; + assert!(patched.success, "PATCH failed: {:?}", patched.error); + assert_eq!(patched.state.sequence_nr, created_sequence + 1); + assert_eq!( + events + .lookup_by_key("default", "Doc", "path", &doc_key_hash("ws", "/old")) + .await + .expect("old key lookup"), + None, + "PATCH must release the old declared key" + ); + assert_eq!( + events + .lookup_by_key("default", "Doc", "path", &doc_key_hash("ws", "/patched"),) + .await + .expect("new key lookup"), + Some("doc-a".to_string()), + "PATCH must claim the new declared key" + ); + + let rejected = update(&first, serde_json::json!({"Path": "/blocked"}), false).await; + assert!( + !rejected.success, + "a conflicting PATCH must be rejected instead of mutating memory only" + ); + assert_eq!(rejected.state.fields["Path"], "/patched"); + assert_eq!(rejected.state.sequence_nr, patched.state.sequence_nr); + + let replaced = update( + &first, + serde_json::json!({"WorkspaceId": "ws", "Path": "/put"}), + true, + ) + .await; + assert!(replaced.success, "PUT failed: {:?}", replaced.error); + assert_eq!(replaced.state.sequence_nr, patched.state.sequence_nr + 1); + assert!(replaced.state.fields.get("Note").is_none()); + assert_eq!(replaced.state.fields["Id"], "doc-a"); + assert_eq!( + events + .lookup_by_key("default", "Doc", "path", &doc_key_hash("ws", "/patched"),) + .await + .expect("patched key lookup"), + None + ); + assert_eq!( + events + .lookup_by_key("default", "Doc", "path", &doc_key_hash("ws", "/put")) + .await + .expect("PUT key lookup"), + Some("doc-a".to_string()) + ); + + drop(first); + drop(blocker); + drop(system); + + let restarted = ActorSystem::new("arn238-field-updates-restarted"); + let recovered = restarted.spawn( + EntityActor::with_persistence( + "Doc", + "doc-a", + table, + serde_json::json!({}), + events, + BackendLabel::Sim, + ) + .with_tenant("default"), + "doc-a", + ); + let recovered = state(&recovered).await; + assert_eq!(recovered.state.fields["Path"], "/put"); + assert_eq!(recovered.state.sequence_nr, replaced.state.sequence_nr); + assert!(recovered.state.fields.get("Note").is_none()); +} + +#[derive(Default)] +struct NoopQueryPlane; + +#[async_trait] +impl QueryPlaneStore for NoopQueryPlane { + async fn upsert_projection( + &self, + _tenant: &str, + _entity_type: &str, + _entity_id: &str, + _status: &str, + _fields: &serde_json::Value, + _state: &serde_json::Value, + _sequence_nr: u64, + ) -> Result<(), PersistenceError> { + Ok(()) + } + + async fn remove_projection( + &self, + _tenant: &str, + _entity_type: &str, + _entity_id: &str, + ) -> Result<(), PersistenceError> { + Ok(()) + } + + async fn query_field_index( + &self, + _tenant: &str, + _entity_type: &str, + _where_clause: &str, + _params: Vec, + ) -> Result>, PersistenceError> { + Ok(None) + } + + async fn load_projection_fields_many( + &self, + _tenant: &str, + _entity_type: &str, + _entity_ids: &[String], + _field_names: &[&str], + ) -> Result>, PersistenceError> { + Ok(None) + } + + async fn projected_entity_counts_by_tenant( + &self, + ) -> Result>, PersistenceError> { + Ok(None) + } +} + +/// The data-only create optimization must preserve the same declared-key +/// co-commit and uniqueness behavior as actor-backed creates. +#[tokio::test] +async fn data_only_create_co_commits_declared_keys() { + let (_guard, _clock, _ids) = install_deterministic_context(239); + let tenant = TenantId::default(); + let sim = SimEventStore::no_faults(239); + let events = BoxedEventStore::new(sim.clone()); + let csdl = parse_csdl(CSDL_XML).expect("CSDL parse"); + let mut registry = SpecRegistry::new(); + registry.register_tenant( + tenant.as_str(), + csdl, + CSDL_XML.to_string(), + &[("Doc", DATA_ONLY_DOC_IOA)], + ); + let mut server = ServerState::from_registry(ActorSystem::new("arn238-data-only"), registry); + server.set_storage_stack(StorageStack::new( + BackendLabel::Sim, + events.clone(), + None, + None, + None, + None, + Some(Arc::new(NoopQueryPlane)), + None, + None, + None, + )); + + let created = server + .try_create_data_only_tenant_entity( + &tenant, + "Doc", + "doc-fast-a", + serde_json::json!({"WorkspaceId": "ws", "Path": "/fast"}), + ) + .await + .expect("data-only create result") + .expect("eligible data-only create"); + assert!(created.success); + assert_eq!( + events + .lookup_by_key("default", "Doc", "path", &doc_key_hash("ws", "/fast")) + .await + .expect("fast-path key lookup"), + Some("doc-fast-a".to_string()), + "the optimized create must not leave a post-watermark entity unindexed" + ); + + let duplicate = server + .try_create_data_only_tenant_entity( + &tenant, + "Doc", + "doc-fast-b", + serde_json::json!({"WorkspaceId": "ws", "Path": "/fast"}), + ) + .await; + assert!( + !matches!(duplicate, Ok(Some(response)) if response.success), + "a duplicate declared key must not create a second data-only entity" + ); + assert!( + events + .read_events("default:Doc:doc-fast-b", 0) + .await + .expect("duplicate journal read") + .is_empty(), + "a rejected duplicate must leave the journal unchanged" + ); +} + +/// Key repair enumerates through a bounded cursor. An entity on page two must +/// be reconciled before the type watermark can become authoritative. +#[tokio::test] +async fn key_repair_reconciles_entities_beyond_the_first_bounded_page() { + let (_guard, _clock, _ids) = install_deterministic_context(24_256); + let tenant = TenantId::default(); + let sim = SimEventStore::no_faults(24_256); + let events = BoxedEventStore::new(sim.clone()); + let csdl = parse_csdl(CSDL_XML).expect("CSDL parse"); + let mut registry = SpecRegistry::new(); + registry.register_tenant( + tenant.as_str(), + csdl, + CSDL_XML.to_string(), + &[("Doc", DATA_ONLY_DOC_IOA)], + ); + let mut server = ServerState::from_registry( + ActorSystem::new("arn238-key-repair-bounded-pages"), + registry, + ); + server.set_storage_stack(StorageStack::new( + BackendLabel::Sim, + events.clone(), + None, + None, + None, + None, + Some(Arc::new(NoopQueryPlane)), + None, + None, + None, + )); + + for index in 0..=256_u16 { + let entity_id = format!("doc-{index:03}"); + let created = EntityEvent { + action: "Created".to_string(), + from_status: String::new(), + to_status: "Ready".to_string(), + timestamp: temper_runtime::scheduler::sim_now(), + params: serde_json::json!({ + "WorkspaceId": "ws-paged", + "Path": format!("/doc-{index:03}"), + }), + idempotency_key: None, + }; + sim.append( + &format!("default:Doc:{entity_id}"), + 0, + &[temper_runtime::persistence::PersistenceEnvelope { + sequence_nr: 0, + event_type: "Created".to_string(), + payload: serde_json::to_value(created).expect("serialize Created event"), + metadata: temper_runtime::persistence::EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: temper_runtime::scheduler::sim_now(), + actor_id: format!("default:Doc:{entity_id}"), + }, + }], + ) + .await + .expect("seed pre-index Doc"); + } + + server.populate_key_index_from_snapshots(&tenant).await; + let key_signature = + declared_key_set_signature(&TransitionTable::from_ioa_source(DATA_ONLY_DOC_IOA).keys); + assert_eq!( + events + .key_index_backfilled_types(tenant.as_str()) + .await + .expect("load key coverage"), + vec![("Doc".to_string(), key_signature)], + "coverage may publish only after every bounded page is reconciled" + ); + for index in [0_u16, 256] { + assert_eq!( + events + .lookup_by_key( + tenant.as_str(), + "Doc", + "path", + &doc_key_hash("ws-paged", &format!("/doc-{index:03}")), + ) + .await + .expect("lookup repaired key"), + Some(format!("doc-{index:03}")), + "entity {index} must be repaired even when it is beyond page one" + ); + } +} + +/// A watermark covers the full ordered key definition, not merely its name. +#[test] +fn key_signature_changes_when_same_named_key_properties_change() { + let original = [DeclaredKey { + name: "path".to_string(), + properties: vec!["WorkspaceId".to_string(), "Path".to_string()], + }]; + let changed = [DeclaredKey { + name: "path".to_string(), + properties: vec!["WorkspaceId".to_string(), "ParentId".to_string()], + }]; + assert_ne!( + declared_key_set_signature(&original), + declared_key_set_signature(&changed), + "a same-name key definition change must invalidate the old coverage watermark" + ); + + let reordered_declarations = [ + DeclaredKey { + name: "z".to_string(), + properties: vec!["Z".to_string()], + }, + DeclaredKey { + name: "a".to_string(), + properties: vec!["A".to_string(), "B".to_string()], + }, + ]; + let same_definitions = [ + DeclaredKey { + name: "a".to_string(), + properties: vec!["A".to_string(), "B".to_string()], + }, + DeclaredKey { + name: "z".to_string(), + properties: vec!["Z".to_string()], + }, + ]; + assert_eq!( + declared_key_set_signature(&reordered_declarations), + declared_key_set_signature(&same_definitions), + "declaration order must not make the coverage signature nondeterministic" + ); + + let names = original + .iter() + .map(|key| key.name.clone()) + .collect::>(); + assert_eq!(names, BTreeSet::from(["path".to_string()])); +} diff --git a/crates/temper-server/tests/key_ownership_write_surfaces/audit_only_repair.rs b/crates/temper-server/tests/key_ownership_write_surfaces/audit_only_repair.rs new file mode 100644 index 000000000..7d1355027 --- /dev/null +++ b/crates/temper-server/tests/key_ownership_write_surfaces/audit_only_repair.rs @@ -0,0 +1,187 @@ +use temper_runtime::persistence::{ + COMPOSITE_EVENT_TYPE, CompositeEvent, EntityKeyRow, EventMetadata, KeyIndexBackfillFence, + PersistenceEnvelope, STATE_MATERIALIZATION_EVENT_TYPE, STATE_MATERIALIZATION_SCHEMA, +}; +use temper_runtime::scheduler::sim_now; + +use super::*; + +fn audit_envelope(persistence_id: &str, entity_id: &str) -> PersistenceEnvelope { + PersistenceEnvelope { + sequence_nr: 0, + event_type: COMPOSITE_EVENT_TYPE.to_string(), + payload: serde_json::to_value(CompositeEvent { + tenant: "default".to_string(), + parent_entity_type: "Doc".to_string(), + parent_entity_id: entity_id.to_string(), + parent_action: "Audit".to_string(), + composite_idempotency_key: format!("audit-{entity_id}"), + intent_hash: String::new(), + sub_writes: Vec::new(), + }) + .expect("serialize composite audit"), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: sim_now(), + actor_id: persistence_id.to_string(), + }, + } +} + +fn materialization_envelope( + persistence_id: &str, + entity_id: &str, + workspace_id: &str, + path: &str, +) -> PersistenceEnvelope { + PersistenceEnvelope { + sequence_nr: 0, + event_type: STATE_MATERIALIZATION_EVENT_TYPE.to_string(), + payload: serde_json::json!({ + "schema": STATE_MATERIALIZATION_SCHEMA, + "state": { + "entity_type": "Doc", + "entity_id": entity_id, + "status": "Ready", + "item_count": 0, + "counters": {}, + "booleans": {}, + "lists": {}, + "fields": { + "Id": entity_id, + "Status": "Ready", + "WorkspaceId": workspace_id, + "Path": path, + }, + "events": [], + "total_event_count": 0, + "events_since_snapshot": 0, + "last_snapshot_sequence_nr": 0, + "sequence_nr": 0, + "processed_idempotency_keys": {}, + } + }), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: sim_now(), + actor_id: persistence_id.to_string(), + }, + } +} + +async fn seed_stale_key( + events: &BoxedEventStore, + entity_id: &str, + sequence_nr: u64, + contract_revision: u64, + key_hash: &str, +) { + events + .backfill_entity_keys( + "default", + "Doc", + entity_id, + sequence_nr, + KeyIndexBackfillFence { + key_set_signature: "v3:stale-contract", + contract_revision, + expected_journal_sequence: sequence_nr, + expected_entity_live: true, + expected_snapshot: None, + }, + &[EntityKeyRow { + key_name: "path".to_string(), + key_hash: key_hash.to_string(), + }], + ) + .await + .expect("seed stale key row"); +} + +#[tokio::test] +async fn key_repair_purges_audit_only_phantom_and_preserves_materialized_owner() { + let (_guard, _clock, _ids) = install_deterministic_context(24_257); + let tenant = TenantId::default(); + let sim = SimEventStore::no_faults(24_257); + let events = BoxedEventStore::new(sim.clone()); + let csdl = parse_csdl(CSDL_XML).expect("CSDL parse"); + let mut registry = SpecRegistry::new(); + registry.register_tenant( + tenant.as_str(), + csdl, + CSDL_XML.to_string(), + &[("Doc", DATA_ONLY_DOC_IOA)], + ); + let mut server = + ServerState::from_registry(ActorSystem::new("arn238-audit-only-key-repair"), registry); + server.set_storage_stack(StorageStack::from_sim(sim.clone(), None)); + + let phantom_id = "audit-only-phantom"; + let phantom_pid = format!("default:Doc:{phantom_id}"); + sim.append(&phantom_pid, 0, &[audit_envelope(&phantom_pid, phantom_id)]) + .await + .expect("seed audit-only phantom"); + + let owner_id = "materialized-owner"; + let owner_pid = format!("default:Doc:{owner_id}"); + sim.append( + &owner_pid, + 0, + &[ + materialization_envelope(&owner_pid, owner_id, "ws-live", "/owned"), + audit_envelope(&owner_pid, owner_id), + ], + ) + .await + .expect("seed materialized owner with audit tail"); + + let stale_revision = events + .begin_key_index_backfill("default", "Doc", "v3:stale-contract") + .await + .expect("begin stale key contract"); + let phantom_stale_hash = doc_key_hash("ws-stale", "/phantom"); + let owner_stale_hash = doc_key_hash("ws-stale", "/owner"); + seed_stale_key(&events, phantom_id, 1, stale_revision, &phantom_stale_hash).await; + seed_stale_key(&events, owner_id, 2, stale_revision, &owner_stale_hash).await; + + server.populate_key_index_from_snapshots(&tenant).await; + + assert_eq!( + events + .lookup_by_key("default", "Doc", "path", &phantom_stale_hash) + .await + .expect("lookup phantom stale key"), + None, + "recognized audit-only phantoms must be source-fenced to zero key rows" + ); + assert_eq!( + events + .lookup_by_key("default", "Doc", "path", &owner_stale_hash) + .await + .expect("lookup materialized stale key"), + None, + "materialized owners must replace stale claims with current fields" + ); + assert_eq!( + events + .lookup_by_key("default", "Doc", "path", &doc_key_hash("ws-live", "/owned"),) + .await + .expect("lookup materialized current key"), + Some(owner_id.to_string()), + "valid state materialization must remain a live declared-key owner" + ); + let current_signature = + declared_key_set_signature(&TransitionTable::from_ioa_source(DATA_ONLY_DOC_IOA).keys); + assert_eq!( + events + .key_index_backfilled_types("default") + .await + .expect("load repaired coverage"), + vec![("Doc".to_string(), current_signature)], + "coverage must publish after the phantom is purged and the owner is repaired" + ); +} diff --git a/crates/temper-server/tests/key_ownership_write_surfaces/field_update_recovery.rs b/crates/temper-server/tests/key_ownership_write_surfaces/field_update_recovery.rs new file mode 100644 index 000000000..6a926bafb --- /dev/null +++ b/crates/temper-server/tests/key_ownership_write_surfaces/field_update_recovery.rs @@ -0,0 +1,1075 @@ +//! Recovery and replay boundaries for durable PATCH/PUT field updates. + +use super::*; +use temper_runtime::persistence::{ + COMPOSITE_EVENT_TYPE, CompositeEvent, EntityKeyRow, EventMetadata, IndexReconciliation, + PersistenceEnvelope, +}; +use temper_runtime::scheduler::{sim_now, sim_uuid}; +use temper_server::entity_actor::types::{ + EntityEvent, MAX_DURABLE_IDEMPOTENCY_KEYS_PER_ENTITY, MAX_EVENTS_SINCE_SNAPSHOT, + RECENT_EVENTS_BUDGET_DEFAULT, +}; + +const FIELD_UPDATE_COLLISION_IOA: &str = r#" +[automaton] +name = "CollisionDoc" +states = ["New", "Ready"] +initial = "New" + +[[state]] +name = "Marker" +type = "string" +initial = "" + +[[action]] +name = "Temper.Internal.FieldUpdate.v1" +kind = "input" +from = ["New"] +to = "Ready" +params = ["Marker"] +"#; + +const BUDGET_DOC_IOA: &str = r#" +[automaton] +name = "BudgetDoc" +states = ["Ready"] +initial = "Ready" + +[[state]] +name = "Value" +type = "string" +initial = "" + +[[action]] +name = "Noop" +kind = "input" +from = ["Ready"] +to = "Ready" +params = ["Value"] +"#; + +/// An in-memory actor retains idempotency tokens beyond its recent-event +/// observability window. Exact PATCH retries must remain verifiable for that +/// entire retained token window, and mismatched reuse must still fail closed. +#[tokio::test] +async fn in_memory_field_update_retry_survives_recent_event_eviction() { + let (_guard, _clock, _ids) = install_deterministic_context(296); + let table = Arc::new(RwLock::new(TransitionTable::from_ioa_source( + BUDGET_DOC_IOA, + ))); + let system = ActorSystem::new("arn238-in-memory-field-update-intent"); + let actor = system.spawn( + EntityActor::new("BudgetDoc", "memory-intent", table, serde_json::json!({})), + "memory-intent", + ); + + let token = "memory-field-update-intent".to_string(); + let fields = serde_json::json!({"Value": "first"}); + let first = update_with_idempotency(&actor, fields.clone(), false, token.clone()).await; + assert!(first.success, "first PATCH failed: {:?}", first.error); + + for index in 0..=RECENT_EVENTS_BUDGET_DEFAULT { + let response = action( + &actor, + "Noop", + serde_json::json!({"Value": format!("later-{index}")}), + ) + .await; + assert!( + response.success, + "Noop {index} failed: {:?}", + response.error + ); + } + let before_retry = state(&actor).await; + assert!( + before_retry + .state + .events + .iter() + .all(|event| event.idempotency_key.as_deref() != Some(token.as_str())), + "the source field-update event must leave the recent-event window" + ); + + let exact_retry = update_with_idempotency(&actor, fields, false, token.clone()).await; + assert!( + exact_retry.success, + "exact PATCH retry failed after recent-event eviction: {:?}", + exact_retry.error + ); + assert_eq!( + exact_retry.state.total_event_count, before_retry.state.total_event_count, + "an exact retry must not append another event" + ); + + let mismatched_retry = update_with_idempotency( + &actor, + serde_json::json!({"Value": "different"}), + false, + token, + ) + .await; + assert!( + !mismatched_retry.success, + "the retained token must remain bound to its original PATCH intent" + ); + assert_eq!( + mismatched_retry.state.total_event_count, + before_retry.state.total_event_count + ); +} + +/// A field-update intent proof must identify the same retained token record, +/// not merely the same token text. Once the token is evicted and rebound to a +/// domain action, its former PATCH intent must not be acknowledged. +#[tokio::test] +async fn in_memory_field_update_retry_rejects_rebound_action_token() { + let (_guard, _clock, _ids) = install_deterministic_context(297); + let table = Arc::new(RwLock::new(TransitionTable::from_ioa_source( + BUDGET_DOC_IOA, + ))); + let system = ActorSystem::new("arn238-in-memory-field-update-rebind"); + let actor = system.spawn( + EntityActor::new("BudgetDoc", "memory-rebind", table, serde_json::json!({})), + "memory-rebind", + ); + + let token = "rebound-field-update-token".to_string(); + let fields = serde_json::json!({"Value": "field-update"}); + let first = update_with_idempotency(&actor, fields.clone(), false, token.clone()).await; + assert!(first.success, "first PATCH failed: {:?}", first.error); + + for index in 0..MAX_DURABLE_IDEMPOTENCY_KEYS_PER_ENTITY { + let response: temper_server::entity_actor::EntityResponse = actor + .ask( + EntityMsg::Action { + name: "Noop".to_string(), + params: serde_json::json!({"Value": format!("evict-{index}")}), + cross_entity_booleans: BTreeMap::new(), + idempotency_key: Some(format!("action-token-{index}")), + }, + Duration::from_secs(5), + ) + .await + .expect("token-eviction action response"); + assert!( + response.success, + "token-eviction action {index} failed: {:?}", + response.error + ); + } + + let rebound: temper_server::entity_actor::EntityResponse = actor + .ask( + EntityMsg::Action { + name: "Noop".to_string(), + params: serde_json::json!({"Value": "rebound-action"}), + cross_entity_booleans: BTreeMap::new(), + idempotency_key: Some(token.clone()), + }, + Duration::from_secs(5), + ) + .await + .expect("rebound action response"); + assert!( + rebound.success, + "rebound action failed: {:?}", + rebound.error + ); + assert_eq!(rebound.state.fields["Value"], "rebound-action"); + + for index in 0..=RECENT_EVENTS_BUDGET_DEFAULT { + let response = action( + &actor, + "Noop", + serde_json::json!({"Value": format!("after-rebind-{index}")}), + ) + .await; + assert!( + response.success, + "post-rebind action {index} failed: {:?}", + response.error + ); + } + let before_retry = state(&actor).await; + assert!( + before_retry + .state + .events + .iter() + .all(|event| event.idempotency_key.as_deref() != Some(token.as_str())), + "the rebound domain event must leave the recent-event window" + ); + + let stale_retry = update_with_idempotency(&actor, fields, false, token).await; + assert!( + !stale_retry.success, + "an old PATCH intent must not match a token rebound to a domain action" + ); + assert_eq!( + stale_retry.state.fields["Value"], + before_retry.state.fields["Value"] + ); + assert_eq!( + stale_retry.state.total_event_count, + before_retry.state.total_event_count + ); +} + +fn event_envelope(persistence_id: &str, event: EntityEvent) -> PersistenceEnvelope { + PersistenceEnvelope { + sequence_nr: 0, + event_type: event.action.clone(), + payload: serde_json::to_value(&event).expect("event serialization"), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: event.timestamp, + actor_id: persistence_id.to_string(), + }, + } +} + +/// A caller token identifies one exact PATCH/PUT intent. Reusing it for a +/// different field set or replacement mode must fail closed while an exact +/// retry remains successful. +#[tokio::test] +async fn field_update_idempotency_rejects_mismatched_live_intent() { + let (_guard, _clock, _ids) = install_deterministic_context(292); + let events = BoxedEventStore::new(SimEventStore::no_faults(292)); + let table = Arc::new(RwLock::new(TransitionTable::from_ioa_source(DOC_IOA))); + let system = ActorSystem::new("arn238-field-update-live-intent"); + let actor = system.spawn( + EntityActor::with_persistence( + "Doc", + "live-intent", + table, + serde_json::json!({}), + events, + BackendLabel::Sim, + ) + .with_tenant("default"), + "live-intent", + ); + + let token = "field-update-intent-live".to_string(); + let fields = serde_json::json!({ + "WorkspaceId": "ws", + "Path": "/first", + "Metadata": {"z": 1, "a": 2} + }); + let first = update_with_idempotency(&actor, fields.clone(), false, token.clone()).await; + assert!(first.success, "first PATCH failed: {:?}", first.error); + + let reordered_fields = serde_json::json!({ + "Metadata": {"a": 2, "z": 1}, + "Path": "/first", + "WorkspaceId": "ws" + }); + let exact_retry = update_with_idempotency(&actor, reordered_fields, false, token.clone()).await; + assert!( + exact_retry.success, + "exact PATCH retry failed: {:?}", + exact_retry.error + ); + assert_eq!(exact_retry.state.sequence_nr, first.state.sequence_nr); + + let different_fields = update_with_idempotency( + &actor, + serde_json::json!({"WorkspaceId": "ws", "Path": "/second"}), + false, + token.clone(), + ) + .await; + assert!( + !different_fields.success, + "a reused token must not acknowledge a different PATCH" + ); + assert_eq!(different_fields.state.fields["Path"], "/first"); + assert_eq!(different_fields.state.sequence_nr, first.state.sequence_nr); + + let different_semantics = update_with_idempotency(&actor, fields, true, token).await; + assert!( + !different_semantics.success, + "a reused token must not acknowledge PATCH as PUT" + ); + assert_eq!(different_semantics.state.fields["Path"], "/first"); + assert_eq!( + different_semantics.state.sequence_nr, + first.state.sequence_nr + ); +} + +/// Durable replay must retain the request-intent binding across actor restart. +#[tokio::test] +async fn field_update_idempotency_rejects_mismatched_intent_after_restart() { + let (_guard, _clock, _ids) = install_deterministic_context(293); + let events = BoxedEventStore::new(SimEventStore::no_faults(293)); + let table = Arc::new(RwLock::new(TransitionTable::from_ioa_source(DOC_IOA))); + let system = ActorSystem::new("arn238-field-update-restart-intent"); + let actor = system.spawn( + EntityActor::with_persistence( + "Doc", + "restart-intent", + table.clone(), + serde_json::json!({}), + events.clone(), + BackendLabel::Sim, + ) + .with_tenant("default"), + "restart-intent", + ); + let token = "field-update-intent-restart".to_string(); + let fields = serde_json::json!({ + "WorkspaceId": "ws", + "Path": "/first", + "Metadata": {"z": 1, "a": 2} + }); + let first = update_with_idempotency(&actor, fields.clone(), false, token.clone()).await; + assert!(first.success, "first PATCH failed: {:?}", first.error); + drop(actor); + drop(system); + + let restarted = ActorSystem::new("arn238-field-update-restart-intent-recovered"); + let recovered = restarted.spawn( + EntityActor::with_persistence( + "Doc", + "restart-intent", + table, + serde_json::json!({}), + events, + BackendLabel::Sim, + ) + .with_tenant("default"), + "restart-intent", + ); + let reordered_fields = serde_json::json!({ + "Metadata": {"a": 2, "z": 1}, + "Path": "/first", + "WorkspaceId": "ws" + }); + let exact_retry = + update_with_idempotency(&recovered, reordered_fields, false, token.clone()).await; + assert!( + exact_retry.success, + "exact PATCH retry failed after restart: {:?}", + exact_retry.error + ); + assert_eq!(exact_retry.state.sequence_nr, first.state.sequence_nr); + + let mismatched = update_with_idempotency( + &recovered, + serde_json::json!({"WorkspaceId": "ws", "Path": "/after-restart"}), + false, + token, + ) + .await; + assert!( + !mismatched.success, + "durable replay must reject a mismatched token reuse" + ); + assert_eq!(mismatched.state.fields["Path"], "/first"); + assert_eq!(mismatched.state.sequence_nr, first.state.sequence_nr); +} + +/// Replay must bind a field-update key to its exact envelope even when an +/// internal audit envelope consumed the preceding journal sequence. +#[tokio::test] +async fn field_update_idempotency_uses_exact_sequence_after_internal_envelope() { + let (_guard, _clock, _ids) = install_deterministic_context(294); + let sim = SimEventStore::no_faults(294); + let events = BoxedEventStore::new(sim.clone()); + let persistence_id = "default:Doc:internal-before-field-update"; + let metadata = |event_id| EventMetadata { + event_id, + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: sim_now(), + actor_id: persistence_id.to_string(), + }; + let token = "field-update-after-internal".to_string(); + let fields = serde_json::json!({"WorkspaceId": "ws", "Path": "/first"}); + events + .append( + persistence_id, + 0, + &[ + PersistenceEnvelope { + sequence_nr: 0, + event_type: COMPOSITE_EVENT_TYPE.to_string(), + payload: serde_json::to_value(CompositeEvent { + tenant: "default".to_string(), + parent_entity_type: "Doc".to_string(), + parent_entity_id: "internal-before-field-update".to_string(), + parent_action: "Audit".to_string(), + composite_idempotency_key: "internal-audit".to_string(), + intent_hash: "audit-intent".to_string(), + sub_writes: vec![], + }) + .expect("serialize composite audit"), + metadata: metadata(sim_uuid()), + }, + PersistenceEnvelope { + sequence_nr: 0, + event_type: "Temper.Internal.FieldUpdate.v1".to_string(), + payload: serde_json::json!({ + "schema": "temper.field-update.v1", + "fields": fields.clone(), + "replace": false, + "idempotency_key": token.clone() + }), + metadata: metadata(sim_uuid()), + }, + ], + ) + .await + .expect("seed internal and field-update envelopes"); + + let table = Arc::new(RwLock::new(TransitionTable::from_ioa_source(DOC_IOA))); + let system = ActorSystem::new("arn238-field-update-exact-sequence"); + let actor = system.spawn( + EntityActor::with_persistence( + "Doc", + "internal-before-field-update", + table, + serde_json::json!({}), + events, + BackendLabel::Sim, + ) + .with_tenant("default"), + "internal-before-field-update", + ); + let exact_retry = update_with_idempotency(&actor, fields, false, token).await; + assert!( + exact_retry.success, + "exact retry must load the field-update envelope: {:?}", + exact_retry.error + ); + assert_eq!(exact_retry.state.sequence_nr, 2); + assert_eq!(sim.dump_journal(persistence_id).len(), 2); +} + +/// A same-sequence snapshot rewrite changes the legacy field baseline without +/// advancing the journal. The next actor write must detect that source change, +/// recover it, and derive key ownership from the replacement fields. +#[tokio::test] +async fn field_update_retries_when_snapshot_generation_changes_without_journal_advance() { + let (_guard, _clock, _ids) = install_deterministic_context(280); + let sim = SimEventStore::no_faults(280); + let events = BoxedEventStore::new(sim.clone()); + let persistence_id = "default:Doc:snapshot-generation-race"; + let legacy_snapshot = |workspace: &str| { + serde_json::to_vec(&serde_json::json!({ + "entity_type": "Doc", + "entity_id": "snapshot-generation-race", + "status": "Ready", + "item_count": 0, + "fields": { + "Id": "snapshot-generation-race", + "Status": "Ready", + "WorkspaceId": workspace, + "Path": "/journal" + } + })) + .expect("serialize legacy snapshot") + }; + events + .save_snapshot(persistence_id, 1, &legacy_snapshot("ws-before")) + .await + .expect("seed first snapshot baseline"); + let timestamp = sim_now(); + events + .append( + persistence_id, + 0, + &[PersistenceEnvelope { + sequence_nr: 1, + event_type: "Temper.Internal.FieldUpdate.v1".to_string(), + payload: serde_json::json!({ + "schema": "temper.field-update.v1", + "fields": {"Path": "/journal"}, + "replace": false, + "idempotency_key": "snapshot-generation-seed" + }), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: persistence_id.to_string(), + }, + }], + ) + .await + .expect("seed equal-sequence journal generation"); + + let table = Arc::new(RwLock::new(TransitionTable::from_ioa_source(DOC_IOA))); + let system = ActorSystem::new("arn238-snapshot-generation-race"); + let actor = system.spawn( + EntityActor::with_persistence( + "Doc", + "snapshot-generation-race", + table, + serde_json::json!({}), + events.clone(), + BackendLabel::Sim, + ) + .with_tenant("default"), + "snapshot-generation-race", + ); + let before_rewrite = state(&actor).await.state; + assert_eq!(before_rewrite.fields["WorkspaceId"], "ws-before"); + + events + .save_snapshot(persistence_id, 1, &legacy_snapshot("ws-after")) + .await + .expect("rewrite the captured snapshot generation"); + + let updated = update(&actor, serde_json::json!({"Path": "/after"}), false).await; + assert!( + updated.success, + "field update retry failed: {:?}", + updated.error + ); + assert_eq!( + updated.state.fields["WorkspaceId"], "ws-after", + "the append must recover the replacement snapshot before deriving state" + ); + assert_eq!(updated.state.fields["Path"], "/after"); + assert_eq!( + events + .lookup_by_key( + "default", + "Doc", + "path", + &doc_key_hash("ws-after", "/after"), + ) + .await + .expect("lookup replacement ownership"), + Some("snapshot-generation-race".to_string()) + ); + assert_eq!( + events + .lookup_by_key( + "default", + "Doc", + "path", + &doc_key_hash("ws-before", "/after"), + ) + .await + .expect("lookup stale ownership"), + None + ); +} + +#[tokio::test] +async fn delete_retries_against_a_same_sequence_snapshot_rewrite() { + let (_guard, _clock, _ids) = install_deterministic_context(287); + let sim = SimEventStore::no_faults(287); + let events = BoxedEventStore::new(sim.clone()); + let persistence_id = "default:Doc:delete-snapshot-generation-race"; + let legacy_snapshot = |workspace: &str| { + serde_json::to_vec(&serde_json::json!({ + "entity_type": "Doc", + "entity_id": "delete-snapshot-generation-race", + "status": "Ready", + "item_count": 0, + "fields": { + "Id": "delete-snapshot-generation-race", + "Status": "Ready", + "WorkspaceId": workspace, + "Path": "/delete-me" + } + })) + .expect("serialize legacy snapshot") + }; + let captured = legacy_snapshot("ws-before-delete"); + events + .save_snapshot(persistence_id, 1, &captured) + .await + .expect("seed captured snapshot"); + let key_set_signature = { + let table = TransitionTable::from_ioa_source(DOC_IOA); + declared_key_set_signature(&table.keys) + }; + events + .append_with_index_rows( + persistence_id, + 0, + &[PersistenceEnvelope { + sequence_nr: 0, + event_type: "Temper.Internal.FieldUpdate.v1".to_string(), + payload: serde_json::json!({ + "schema": "temper.field-update.v1", + "fields": {"Path": "/delete-me"}, + "replace": false, + "idempotency_key": "delete-source-seed" + }), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: sim_now(), + actor_id: persistence_id.to_string(), + }, + }], + &[EntityKeyRow { + key_name: "path".to_string(), + key_hash: doc_key_hash("ws-before-delete", "/delete-me"), + }], + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(key_set_signature), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .expect("seed equal-sequence journal and key row"); + + let table = Arc::new(RwLock::new(TransitionTable::from_ioa_source(DOC_IOA))); + let system = ActorSystem::new("arn238-delete-snapshot-generation-race"); + let actor = system.spawn( + EntityActor::with_persistence( + "Doc", + "delete-snapshot-generation-race", + table, + serde_json::json!({}), + events.clone(), + BackendLabel::Sim, + ) + .with_tenant("default"), + "delete-snapshot-generation-race", + ); + assert_eq!( + state(&actor).await.state.fields["WorkspaceId"], + "ws-before-delete" + ); + events + .save_snapshot(persistence_id, 1, &legacy_snapshot("ws-after-delete")) + .await + .expect("replace captured snapshot generation"); + + let deleted: EntityResponse = actor + .ask(EntityMsg::Delete, Duration::from_secs(5)) + .await + .expect("delete response"); + assert!(deleted.success, "delete failed: {:?}", deleted.error); + assert_eq!(deleted.state.status, "Deleted"); + assert_eq!(deleted.state.sequence_nr, 2); + assert_eq!(deleted.state.fields["WorkspaceId"], "ws-after-delete"); + assert_eq!(sim.dump_journal(persistence_id).len(), 2); + assert_eq!( + events + .lookup_by_key( + "default", + "Doc", + "path", + &doc_key_hash("ws-before-delete", "/delete-me"), + ) + .await + .expect("lookup purged key"), + None + ); +} + +/// A real intervening journal append must make PATCH rebuild from the durable +/// stream before retrying. The retried update keeps the other writer's fields, +/// moves exact key ownership, and leaves the actor fresh for the next message. +#[tokio::test] +async fn field_update_replays_real_concurrent_append_before_retry() { + let (_guard, _clock, _ids) = install_deterministic_context(240); + let sim = SimEventStore::no_faults(240); + let events = BoxedEventStore::new(sim.clone()); + let table = Arc::new(RwLock::new(TransitionTable::from_ioa_source(DOC_IOA))); + let system = ActorSystem::new("arn238-field-update-real-race"); + let actor = system.spawn( + EntityActor::with_persistence( + "Doc", + "doc-race", + table.clone(), + serde_json::json!({}), + events.clone(), + BackendLabel::Sim, + ) + .with_tenant("default"), + "doc-race", + ); + + assert!( + action( + &actor, + "Create", + serde_json::json!({"WorkspaceId": "ws", "Path": "/before"}), + ) + .await + .success + ); + let before_race = state(&actor).await.state; + let persistence_id = "default:Doc:doc-race"; + let external_event = EntityEvent { + action: "Rekey".to_string(), + from_status: "Ready".to_string(), + to_status: "Ready".to_string(), + timestamp: sim_now(), + params: serde_json::json!({ + "WorkspaceId": "ws", + "Path": "/external", + "ExternalOnly": "must-survive" + }), + idempotency_key: None, + }; + let key_set_signature = { + let table = table.read().expect("table lock"); + declared_key_set_signature(&table.keys) + }; + events + .append_with_index_rows( + persistence_id, + before_race.sequence_nr, + &[event_envelope(persistence_id, external_event)], + &[EntityKeyRow { + key_name: "path".to_string(), + key_hash: doc_key_hash("ws", "/external"), + }], + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(key_set_signature), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .expect("intervening writer append"); + + let retried = update(&actor, serde_json::json!({"Path": "/after-race"}), false).await; + assert!(retried.success, "PATCH retry failed: {:?}", retried.error); + assert_eq!(retried.state.sequence_nr, before_race.sequence_nr + 2); + assert_eq!(retried.state.fields["ExternalOnly"], "must-survive"); + assert_eq!(retried.state.fields["Path"], "/after-race"); + assert_eq!( + events + .lookup_by_key("default", "Doc", "path", &doc_key_hash("ws", "/external"),) + .await + .expect("external key lookup"), + None + ); + assert_eq!( + events + .lookup_by_key("default", "Doc", "path", &doc_key_hash("ws", "/after-race"),) + .await + .expect("retried key lookup"), + Some("doc-race".to_string()) + ); + + let follow_up = update( + &actor, + serde_json::json!({"Path": "/after-follow-up"}), + false, + ) + .await; + assert!( + follow_up.success, + "actor remained stale after retry: {:?}", + follow_up.error + ); + assert_eq!(follow_up.state.sequence_nr, before_race.sequence_nr + 3); + assert_eq!(follow_up.state.fields["ExternalOnly"], "must-survive"); +} + +/// If catch-up discovers that another writer committed the terminal tombstone, +/// PATCH must reject instead of appending an unreplayable suffix after `Deleted`. +#[tokio::test] +async fn field_update_rejects_concurrent_delete_and_restarts_at_tombstone() { + let (_guard, _clock, _ids) = install_deterministic_context(244); + let sim = SimEventStore::no_faults(244); + let events = BoxedEventStore::new(sim.clone()); + let table = Arc::new(RwLock::new(TransitionTable::from_ioa_source(DOC_IOA))); + let system = ActorSystem::new("arn238-field-update-delete-race"); + let actor = system.spawn( + EntityActor::with_persistence( + "Doc", + "doc-deleted", + table.clone(), + serde_json::json!({}), + events.clone(), + BackendLabel::Sim, + ) + .with_tenant("default"), + "doc-deleted", + ); + assert!( + action( + &actor, + "Create", + serde_json::json!({"WorkspaceId": "ws", "Path": "/deleted"}), + ) + .await + .success + ); + let created = state(&actor).await.state; + let persistence_id = "default:Doc:doc-deleted"; + let tombstone = EntityEvent { + action: "Deleted".to_string(), + from_status: created.status.clone(), + to_status: "Deleted".to_string(), + timestamp: sim_now(), + params: serde_json::json!({}), + idempotency_key: None, + }; + let signature = declared_key_set_signature(&table.read().expect("table lock").keys); + events + .append_with_index_rows( + persistence_id, + created.sequence_nr, + &[event_envelope(persistence_id, tombstone)], + &[], + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(signature), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .expect("external tombstone append"); + let tombstone_sequence = created.sequence_nr + 1; + assert_eq!( + sim.dump_journal(persistence_id).len(), + tombstone_sequence as usize, + "precondition: the external writer committed only the tombstone" + ); + + let rejected = update( + &actor, + serde_json::json!({"Path": "/must-not-append"}), + false, + ) + .await; + assert!(!rejected.success); + assert_eq!( + rejected.error.as_deref(), + Some("cannot update a deleted entity") + ); + assert_eq!(rejected.state.status, "Deleted"); + assert_eq!(rejected.state.sequence_nr, tombstone_sequence); + assert_eq!( + sim.dump_journal(persistence_id).len(), + tombstone_sequence as usize, + "rejected PATCH must not append after the tombstone" + ); + + drop(actor); + drop(system); + let restarted = ActorSystem::new("arn238-field-update-delete-race-restart"); + let recovered = restarted.spawn( + EntityActor::with_persistence( + "Doc", + "doc-deleted", + table, + serde_json::json!({}), + events, + BackendLabel::Sim, + ) + .with_tenant("default"), + "doc-deleted", + ); + let recovered = state(&recovered).await; + assert_eq!(recovered.state.status, "Deleted"); + assert_eq!(recovered.state.sequence_nr, tombstone_sequence); + assert_eq!( + sim.dump_journal(persistence_id).len(), + tombstone_sequence as usize + ); +} + +/// The private PATCH/PUT event type is not a reserved spec action name. A +/// normal action with that exact name carries an EntityEvent payload and must +/// replay through the transition table after restart. +#[tokio::test] +async fn internal_field_update_event_type_does_not_shadow_spec_action() { + let (_guard, _clock, _ids) = install_deterministic_context(241); + let sim = SimEventStore::no_faults(241); + let events = BoxedEventStore::new(sim); + let table = Arc::new(RwLock::new(TransitionTable::from_ioa_source( + FIELD_UPDATE_COLLISION_IOA, + ))); + let system = ActorSystem::new("arn238-field-update-action-collision"); + let actor = system.spawn( + EntityActor::with_persistence( + "CollisionDoc", + "collision", + table.clone(), + serde_json::json!({}), + events.clone(), + BackendLabel::Sim, + ) + .with_tenant("default"), + "collision", + ); + + let dispatched = action( + &actor, + "Temper.Internal.FieldUpdate.v1", + serde_json::json!({"Marker": "ordinary-action"}), + ) + .await; + assert!(dispatched.success, "collision action failed"); + assert_eq!(dispatched.state.status, "Ready"); + drop(actor); + drop(system); + + let restarted = ActorSystem::new("arn238-field-update-action-collision-restart"); + let recovered = restarted.spawn( + EntityActor::with_persistence( + "CollisionDoc", + "collision", + table, + serde_json::json!({}), + events, + BackendLabel::Sim, + ) + .with_tenant("default"), + "collision", + ); + let recovered = state(&recovered).await; + assert_eq!(recovered.state.status, "Ready"); + assert_eq!(recovered.state.fields["Marker"], "ordinary-action"); +} + +/// PATCH/PUT and terminal Delete share the bounded raw replay-tail budget with +/// spec actions. A legacy snapshot must not reset that reconstructed budget; at +/// the exact cap both writes refuse to append and restart remains bounded. +#[tokio::test] +async fn field_update_respects_replay_tail_budget_and_restart_boundary() { + let (_guard, _clock, _ids) = install_deterministic_context(242); + let sim = SimEventStore::no_faults(242); + let events = BoxedEventStore::new(sim.clone()); + let persistence_id = "default:BudgetDoc:budget"; + let envelopes = (0..MAX_EVENTS_SINCE_SNAPSHOT) + .map(|index| { + event_envelope( + persistence_id, + EntityEvent { + action: "Noop".to_string(), + from_status: "Ready".to_string(), + to_status: "Ready".to_string(), + timestamp: sim_now(), + params: serde_json::json!({"Value": index.to_string()}), + idempotency_key: None, + }, + ) + }) + .collect::>(); + events + .append(persistence_id, 0, &envelopes) + .await + .expect("seed replay-tail boundary"); + let legacy_snapshot = serde_json::to_vec(&serde_json::json!({ + "entity_type": "BudgetDoc", + "entity_id": "budget", + "status": "Ready", + "item_count": 0, + "counters": {}, + "booleans": {}, + "lists": {}, + "fields": {"Id": "budget", "Status": "Ready", "Value": "legacy"}, + "events": [], + "total_event_count": 5, + "events_since_snapshot": 0, + "last_snapshot_sequence_nr": 5, + "sequence_nr": 5, + "processed_idempotency_keys": {} + })) + .expect("serialize legacy snapshot"); + events + .save_snapshot(persistence_id, 5, &legacy_snapshot) + .await + .expect("seed untrusted legacy snapshot"); + + let table = Arc::new(RwLock::new(TransitionTable::from_ioa_source( + BUDGET_DOC_IOA, + ))); + let system = ActorSystem::new("arn238-field-update-budget"); + let actor = system.spawn( + EntityActor::with_persistence( + "BudgetDoc", + "budget", + table.clone(), + serde_json::json!({}), + events.clone(), + BackendLabel::Sim, + ) + .with_tenant("default"), + "budget", + ); + let at_boundary = state(&actor).await; + assert_eq!( + at_boundary.state.events_since_snapshot, + MAX_EVENTS_SINCE_SNAPSHOT + ); + + let rejected = update( + &actor, + serde_json::json!({"Value": "must-not-append"}), + false, + ) + .await; + assert!(!rejected.success); + assert!( + rejected + .error + .as_deref() + .is_some_and(|error| error.contains("Event budget exhausted")) + ); + assert_eq!( + sim.dump_journal(persistence_id).len(), + MAX_EVENTS_SINCE_SNAPSHOT + ); + let rejected_delete: temper_server::entity_actor::EntityResponse = actor + .ask(EntityMsg::Delete, Duration::from_secs(5)) + .await + .expect("delete response"); + assert!(!rejected_delete.success); + assert!( + rejected_delete + .error + .as_deref() + .is_some_and(|error| error.contains("Event budget exhausted")) + ); + assert_ne!(rejected_delete.state.status, "Deleted"); + assert_eq!( + sim.dump_journal(persistence_id).len(), + MAX_EVENTS_SINCE_SNAPSHOT + ); + drop(actor); + drop(system); + + let restarted = ActorSystem::new("arn238-field-update-budget-restart"); + let recovered = restarted.spawn( + EntityActor::with_persistence( + "BudgetDoc", + "budget", + table, + serde_json::json!({}), + events, + BackendLabel::Sim, + ) + .with_tenant("default"), + "budget", + ); + let recovered = state(&recovered).await; + assert_eq!( + recovered.state.events_since_snapshot, + MAX_EVENTS_SINCE_SNAPSHOT + ); + assert_eq!( + recovered.state.sequence_nr, + MAX_EVENTS_SINCE_SNAPSHOT as u64 + ); +} diff --git a/crates/temper-server/tests/key_ownership_write_surfaces/key_contract_aba.rs b/crates/temper-server/tests/key_ownership_write_surfaces/key_contract_aba.rs new file mode 100644 index 000000000..67f3c3d72 --- /dev/null +++ b/crates/temper-server/tests/key_ownership_write_surfaces/key_contract_aba.rs @@ -0,0 +1,1177 @@ +//! ABA-safe key-contract watermark regressions. + +use super::*; + +const DOC_IOA_WITHOUT_KEYS: &str = r#" +[automaton] +name = "Doc" +states = ["New", "Ready"] +initial = "New" + +[[state]] +name = "WorkspaceId" +type = "string" +initial = "" + +[[state]] +name = "Path" +type = "string" +initial = "" + +[[action]] +name = "Create" +kind = "input" +from = ["New"] +to = "Ready" +params = ["WorkspaceId", "Path"] + +[[action]] +name = "Rekey" +kind = "input" +from = ["Ready"] +to = "Ready" +params = ["WorkspaceId", "Path"] +"#; + +async fn persist_doc_generation( + platform: &temper_server::platform_store::SimPlatformStore, + tenant: &TenantId, + ioa_source: &str, +) -> Result, String> { + use temper_server::platform_store::{ + PlatformStore, SpecPublication, SpecPublicationMode, TenantConstraintsPublication, + TenantPolicyPublication, + }; + + let content_hash = temper_store_turso::spec_content_hash(ioa_source); + PlatformStore::publish_specs( + platform, + tenant.as_str(), + &[SpecPublication { + entity_type: "Doc", + ioa_source, + csdl_xml: CSDL_XML, + content_hash: &content_hash, + }], + SpecPublicationMode::Replace, + TenantConstraintsPublication::Preserve, + TenantPolicyPublication::Preserve, + None, + None, + &[], + ) + .await +} + +fn doc_publication_intent(ioa_source: &str) -> String { + ServerState::spec_publication_intent( + "test-doc-replace", + [ + ("csdl", CSDL_XML.as_bytes()), + ("spec:Doc", ioa_source.as_bytes()), + ], + ) +} + +fn unrelated_task_publication_intent() -> String { + ServerState::spec_publication_intent( + "test-task-merge", + [ + ("csdl", CSDL_XML.as_bytes()), + ( + "spec:Task", + b"[automaton]\nname = \"Task\"\nstates = [\"Ready\"]\ninitial = \"Ready\"\n", + ), + ], + ) +} + +async fn publish_doc_generation( + server: &ServerState, + platform: &temper_server::platform_store::SimPlatformStore, + publication_guard: &mut temper_server::state::SpecPublicationGuard, + tenant: &TenantId, + ioa_source: &str, +) { + server + .arm_spec_publication( + publication_guard, + tenant, + &doc_publication_intent(ioa_source), + ) + .expect("arm Doc publication"); + persist_doc_generation(platform, tenant, ioa_source) + .await + .expect("publish durable Doc generation"); + let mut cutover = server + .prepare_key_index_contracts_for_spec_activation( + publication_guard, + tenant, + &[("Doc", ioa_source)], + ) + .await + .expect("prepare Doc contract"); + server + .registry + .write() + .expect("registry lock") + .try_register_tenant_with_reactions_constraints_and_key_epochs( + tenant.as_str(), + parse_csdl(CSDL_XML).expect("CSDL parse"), + CSDL_XML.to_string(), + &[("Doc", ioa_source)], + Vec::new(), + None, + false, + &cutover.activation_epochs, + ) + .expect("publish live Doc generation"); + server + .finish_key_index_contract_activation(publication_guard, tenant, &mut cutover) + .await + .expect("finish Doc contract"); + server + .complete_spec_publication_retry(publication_guard, tenant) + .expect("complete Doc publication"); +} + +/// An action must carry the exact table/epoch it used for evaluation through +/// the durable append. Re-reading the live table at append time would stamp an +/// old-A transition with the new A epoch after A -> none -> A and resurrect a +/// row purged by the intermediate empty contract. +#[tokio::test] +async fn delayed_actor_action_cannot_borrow_the_reactivated_a_epoch() { + let (_guard, _clock, _ids) = install_deterministic_context(298); + let sim = SimEventStore::no_faults(298); + let events = BoxedEventStore::new(sim.clone()); + let mut original_table = TransitionTable::from_ioa_source(DOC_IOA); + let signature_a = declared_key_set_signature(&original_table.keys); + let old_epoch = events + .activate_key_index_contract("default", "Doc", &signature_a, false) + .await + .expect("activate original A contract"); + events + .mark_key_index_backfilled("default", "Doc", &signature_a) + .await + .expect("publish original A readiness"); + original_table.key_contract_activation_epoch = old_epoch; + let table = Arc::new(RwLock::new(original_table)); + let actor = ActorSystem::new("arn238-actor-activation-epoch").spawn( + EntityActor::with_persistence( + "Doc", + "actor-epoch-owner", + table.clone(), + serde_json::json!({}), + events.clone(), + BackendLabel::Sim, + ) + .with_tenant("default"), + "actor-epoch-owner", + ); + assert!( + action( + &actor, + "Create", + serde_json::json!({"WorkspaceId": "ws", "Path": "/original"}), + ) + .await + .success + ); + let persistence_id = "default:Doc:actor-epoch-owner"; + let journal_before = sim.dump_journal(persistence_id).len(); + let pause = sim.inject_precommit_append_pause(persistence_id); + let action_future = action( + &actor, + "Rekey", + serde_json::json!({"WorkspaceId": "ws", "Path": "/must-not-resurrect"}), + ); + tokio::pin!(action_future); + tokio::select! { + response = &mut action_future => panic!("action crossed pre-commit barrier: {response:?}"), + () = pause.wait_until_reached() => {} + } + + let empty_table = TransitionTable::from_ioa_source(DOC_IOA_WITHOUT_KEYS); + let signature_none = declared_key_set_signature(&empty_table.keys); + let empty_epoch = events + .activate_key_index_contract("default", "Doc", &signature_none, true) + .await + .expect("activate empty contract"); + let mut empty_table = empty_table; + empty_table.key_contract_activation_epoch = empty_epoch; + *table.write().expect("table lock") = empty_table; + + let mut reactivated_table = TransitionTable::from_ioa_source(DOC_IOA); + let current_epoch = events + .activate_key_index_contract("default", "Doc", &signature_a, false) + .await + .expect("reactivate A contract"); + assert!(current_epoch > old_epoch); + reactivated_table.key_contract_activation_epoch = current_epoch; + *table.write().expect("table lock") = reactivated_table; + + pause.resume(); + let response = action_future.await; + assert!( + !response.success, + "stale evaluated action unexpectedly committed" + ); + assert!( + response + .error + .as_deref() + .is_some_and(|error| error.contains("activation is stale")), + "unexpected stale-action error: {:?}", + response.error + ); + assert_eq!(sim.dump_journal(persistence_id).len(), journal_before); + assert_eq!( + events + .lookup_by_key( + "default", + "Doc", + "path", + &doc_key_hash("ws", "/must-not-resurrect"), + ) + .await + .expect("lookup rejected claim"), + None + ); +} + +/// A coverage watermark is tied to a monotonic key-contract revision, not only +/// the signature text. Cycling A -> no keys -> A must invalidate the original +/// A watermark, and a backfill that started before either live change cannot +/// publish after the corresponding revision has advanced. +#[tokio::test] +async fn key_contract_revision_fences_aba_spec_cycles() { + let (_guard, _clock, _ids) = install_deterministic_context(243); + let sim = SimEventStore::no_faults(243); + let events = BoxedEventStore::new(sim); + let keyed_table = TransitionTable::from_ioa_source(DOC_IOA); + let signature_a = declared_key_set_signature(&keyed_table.keys); + let table = Arc::new(RwLock::new(keyed_table)); + let system = ActorSystem::new("arn238-key-contract-aba"); + let actor = system.spawn( + EntityActor::with_persistence( + "Doc", + "doc-aba", + table.clone(), + serde_json::json!({}), + events.clone(), + BackendLabel::Sim, + ) + .with_tenant("default"), + "doc-aba", + ); + assert!( + action( + &actor, + "Create", + serde_json::json!({"WorkspaceId": "ws", "Path": "/a"}), + ) + .await + .success + ); + + events + .mark_key_index_backfilled("default", "Doc", &signature_a) + .await + .expect("mark A coverage"); + let revision_a = events + .key_index_reconciliation_revision("default", "Doc") + .await + .expect("read A revision"); + assert_eq!( + events + .key_index_backfilled_types("default") + .await + .expect("read A coverage"), + vec![("Doc".to_string(), signature_a.clone())] + ); + + let signature_without_keys = + declared_key_set_signature(&TransitionTable::from_ioa_source(DOC_IOA_WITHOUT_KEYS).keys); + let stale_no_key_repair_revision = events + .begin_key_index_backfill("default", "Doc", &signature_without_keys) + .await + .expect("begin no-key repair"); + assert!(stale_no_key_repair_revision > revision_a); + let concurrent_a = update( + &actor, + serde_json::json!({"Path": "/a-during-no-key-repair"}), + false, + ) + .await; + assert!(concurrent_a.success, "concurrent A write failed"); + let revision_after_concurrent_a = events + .key_index_reconciliation_revision("default", "Doc") + .await + .expect("read post-race A revision"); + assert!(revision_after_concurrent_a > stale_no_key_repair_revision); + assert!( + !events + .mark_key_index_backfilled_if_revision( + "default", + "Doc", + &signature_without_keys, + stale_no_key_repair_revision, + ) + .await + .expect("reject mixed-contract repair"), + "a live A write during a no-key repair must fence publication" + ); + + let without_keys = TransitionTable::from_ioa_source(DOC_IOA_WITHOUT_KEYS); + *table.write().expect("table lock") = without_keys; + let no_key_write = update(&actor, serde_json::json!({"Path": "/while-unkeyed"}), false).await; + assert!(no_key_write.success, "unkeyed write failed"); + let revision_without_keys = events + .key_index_reconciliation_revision("default", "Doc") + .await + .expect("read no-key revision"); + assert!(revision_without_keys > revision_after_concurrent_a); + assert!( + events + .key_index_backfilled_types("default") + .await + .expect("read invalidated coverage") + .is_empty(), + "a live no-key write must invalidate the A watermark" + ); + assert_eq!( + events + .lookup_by_key( + "default", + "Doc", + "path", + &doc_key_hash("ws", "/a-during-no-key-repair"), + ) + .await + .expect("released live-write key lookup"), + None, + "a live write under the empty contract must release prior ownership" + ); + assert!( + !events + .mark_key_index_backfilled_if_revision("default", "Doc", &signature_a, revision_a,) + .await + .expect("reject stale A backfill"), + "a backfill captured under the first A contract must be fenced" + ); + + let restored_table = TransitionTable::from_ioa_source(DOC_IOA); + assert_eq!( + declared_key_set_signature(&restored_table.keys), + signature_a, + "the restored contract intentionally reuses the original signature" + ); + *table.write().expect("table lock") = restored_table; + assert!( + events + .key_index_backfilled_types("default") + .await + .expect("read pre-write restored coverage") + .is_empty(), + "restoring the same signature in memory must not resurrect coverage" + ); + let restored = update(&actor, serde_json::json!({"Path": "/restored-a"}), false).await; + assert!(restored.success, "restored A write failed"); + let restored_revision = events + .key_index_reconciliation_revision("default", "Doc") + .await + .expect("read restored A revision"); + assert!(restored_revision > revision_without_keys); + assert!( + !events + .mark_key_index_backfilled_if_revision( + "default", + "Doc", + &signature_without_keys, + revision_without_keys, + ) + .await + .expect("reject stale no-key backfill"), + "a backfill captured before A was restored must be fenced" + ); + assert!( + events + .key_index_backfilled_types("default") + .await + .expect("read final coverage") + .is_empty(), + "the A -> no-key -> A cycle requires a fresh successful backfill" + ); + assert_eq!( + events + .lookup_by_key("default", "Doc", "path", &doc_key_hash("ws", "/restored-a"),) + .await + .expect("restored A key lookup"), + Some("doc-aba".to_string()) + ); +} + +/// Contract activation must purge and fence A -> none -> A without relying on +/// either an entity write or the detached backfill between the two spec swaps. +/// Re-adding A must not resurrect its old watermark or prevent a new live owner +/// from claiming the released value. +#[tokio::test] +async fn empty_key_contract_backfill_fences_no_write_aba_and_releases_ownership() { + let (_guard, _clock, _ids) = install_deterministic_context(295); + let tenant = TenantId::default(); + let sim = SimEventStore::no_faults(295); + let events = BoxedEventStore::new(sim); + let csdl = parse_csdl(CSDL_XML).expect("CSDL parse"); + let mut registry = SpecRegistry::new(); + registry.register_tenant( + tenant.as_str(), + csdl, + CSDL_XML.to_string(), + &[("Doc", DOC_IOA)], + ); + let mut server = ServerState::from_registry( + ActorSystem::new("arn238-key-contract-no-write-aba"), + registry, + ); + server.set_storage_stack(StorageStack::new( + BackendLabel::Sim, + events.clone(), + None, + None, + None, + None, + None, + None, + None, + None, + )); + + let initial_table = server + .registry + .read() + .expect("registry lock") + .get_table_live(&tenant, "Doc") + .expect("Doc table"); + let original = server.actor_system.spawn( + EntityActor::with_persistence( + "Doc", + "doc-original-owner", + initial_table, + serde_json::json!({}), + events.clone(), + BackendLabel::Sim, + ) + .with_tenant(tenant.as_str()), + "doc-original-owner", + ); + assert!( + action( + &original, + "Create", + serde_json::json!({"WorkspaceId": "ws", "Path": "/released"}), + ) + .await + .success + ); + server.populate_key_index_from_snapshots(&tenant).await; + let signature_a = declared_key_set_signature(&TransitionTable::from_ioa_source(DOC_IOA).keys); + assert_eq!( + events + .key_index_backfilled_types(tenant.as_str()) + .await + .expect("read initial watermark"), + vec![("Doc".to_string(), signature_a.clone())] + ); + + let mut empty_publication = server + .begin_spec_publication(&tenant) + .await + .expect("acquire empty publication"); + server + .arm_spec_publication( + &mut empty_publication, + &tenant, + &doc_publication_intent(DOC_IOA_WITHOUT_KEYS), + ) + .expect("arm empty publication"); + let mut empty_cutover = server + .prepare_key_index_contracts_for_spec_activation( + &empty_publication, + &tenant, + &[("Doc", DOC_IOA_WITHOUT_KEYS)], + ) + .await + .expect("prepare empty key contract"); + server + .registry + .write() + .expect("registry lock") + .try_register_tenant_with_reactions_constraints_and_key_epochs( + tenant.as_str(), + parse_csdl(CSDL_XML).expect("CSDL parse"), + CSDL_XML.to_string(), + &[("Doc", DOC_IOA_WITHOUT_KEYS)], + Vec::new(), + None, + false, + &empty_cutover.activation_epochs, + ) + .expect("publish empty key contract"); + server + .finish_key_index_contract_activation(&mut empty_publication, &tenant, &mut empty_cutover) + .await + .expect("finish empty key contract"); + server + .complete_spec_publication_retry(&mut empty_publication, &tenant) + .expect("complete empty publication"); + assert_eq!( + events + .lookup_by_key( + tenant.as_str(), + "Doc", + "path", + &doc_key_hash("ws", "/released"), + ) + .await + .expect("released key lookup"), + None, + "the empty contract must purge ownership without waiting for a live write" + ); + assert_eq!( + events + .key_index_backfilled_types(tenant.as_str()) + .await + .expect("read empty-contract watermark"), + vec![( + "Doc".to_string(), + declared_key_set_signature( + &TransitionTable::from_ioa_source(DOC_IOA_WITHOUT_KEYS).keys + ), + )], + "the empty contract must become ready only after candidate repair" + ); + + drop(empty_publication); + let mut restored_publication = server + .begin_spec_publication(&tenant) + .await + .expect("acquire restored publication"); + server + .arm_spec_publication( + &mut restored_publication, + &tenant, + &doc_publication_intent(DOC_IOA), + ) + .expect("arm restored publication"); + let mut restored_cutover = server + .prepare_key_index_contracts_for_spec_activation( + &restored_publication, + &tenant, + &[("Doc", DOC_IOA)], + ) + .await + .expect("prepare restored A contract"); + server + .registry + .write() + .expect("registry lock") + .try_register_tenant_with_reactions_constraints_and_key_epochs( + tenant.as_str(), + parse_csdl(CSDL_XML).expect("CSDL parse"), + CSDL_XML.to_string(), + &[("Doc", DOC_IOA)], + Vec::new(), + None, + false, + &restored_cutover.activation_epochs, + ) + .expect("publish restored A contract"); + server + .finish_key_index_contract_activation( + &mut restored_publication, + &tenant, + &mut restored_cutover, + ) + .await + .expect("finish restored A contract"); + server + .complete_spec_publication_retry(&mut restored_publication, &tenant) + .expect("complete restored publication"); + assert_eq!( + events + .key_index_backfilled_types(tenant.as_str()) + .await + .expect("read pre-write restored watermark"), + vec![("Doc".to_string(), signature_a)], + "re-adding A must rebuild the original entity's ownership before opening writes" + ); + let restored_table = server + .registry + .read() + .expect("registry lock") + .get_table_live(&tenant, "Doc") + .expect("restored Doc table"); + let reclaimer = server.actor_system.spawn( + EntityActor::with_persistence( + "Doc", + "doc-new-owner", + restored_table, + serde_json::json!({}), + events.clone(), + BackendLabel::Sim, + ) + .with_tenant(tenant.as_str()), + "doc-new-owner", + ); + let reclaimed = action( + &reclaimer, + "Create", + serde_json::json!({"WorkspaceId": "ws", "Path": "/released"}), + ) + .await; + assert!(!reclaimed.success, "duplicate claimant unexpectedly won"); + assert_eq!( + events + .lookup_by_key( + tenant.as_str(), + "Doc", + "path", + &doc_key_hash("ws", "/released"), + ) + .await + .expect("reclaimed key lookup"), + Some("doc-original-owner".to_string()) + ); +} + +/// Re-activating the exact durable spec advances only the writer epoch. Its +/// retained key-coverage proof must be fenced by revision CAS, not rebuilt by an +/// O(all entities) startup replay. +#[tokio::test] +async fn unchanged_ready_activation_reuses_coverage_without_replaying_entities() { + let (_guard, _clock, _ids) = install_deterministic_context(296); + let tenant = TenantId::default(); + let sim = SimEventStore::no_faults(296); + let events = BoxedEventStore::new(sim.clone()); + let csdl = parse_csdl(CSDL_XML).expect("CSDL parse"); + let mut registry = SpecRegistry::new(); + registry.register_tenant( + tenant.as_str(), + csdl, + CSDL_XML.to_string(), + &[("Doc", DOC_IOA)], + ); + let mut server = ServerState::from_registry( + ActorSystem::new("arn238-retained-activation-coverage"), + registry, + ); + server.set_storage_stack(StorageStack::new( + BackendLabel::Sim, + events.clone(), + None, + None, + None, + None, + None, + None, + None, + None, + )); + + let table = server + .registry + .read() + .expect("registry lock") + .get_table_live(&tenant, "Doc") + .expect("Doc table"); + let actor = server.actor_system.spawn( + EntityActor::with_persistence( + "Doc", + "doc-retained", + table, + serde_json::json!({}), + events.clone(), + BackendLabel::Sim, + ) + .with_tenant(tenant.as_str()), + "doc-retained", + ); + assert!( + action( + &actor, + "Create", + serde_json::json!({"WorkspaceId": "ws", "Path": "/retained"}), + ) + .await + .success + ); + server.populate_key_index_from_snapshots(&tenant).await; + + // The first activation establishes the fingerprint/epoch and legitimately + // performs a repair. The second activation is byte-for-byte identical. + server + .activate_registered_key_contracts(&tenant) + .await + .expect("establish initial activated contract"); + let persistence_id = format!("{tenant}:Doc:doc-retained"); + sim.fail_next_reads(&persistence_id, 1); + + server + .activate_registered_key_contracts(&tenant) + .await + .expect("unchanged activation reuses retained coverage"); + + let read_error = events + .read_events(&persistence_id, 0) + .await + .expect_err("activation should not consume the injected entity read fault"); + assert!( + read_error.to_string().contains("injected read failure"), + "unexpected retained read fault: {read_error}" + ); +} + +/// Durable publication and runtime cutover are one per-tenant critical +/// section. A second publication cannot persist B while A still owns the +/// coordinator, so durable specs, the live registry, and the active key +/// contract finish on the same generation. +#[tokio::test] +async fn concurrent_publications_serialize_persistence_through_runtime_cutover() { + use temper_server::platform_store::{PlatformStore, SimPlatformStore}; + + let (_guard, _clock, _ids) = install_deterministic_context(297); + let tenant = TenantId::default(); + let events = SimEventStore::no_faults(297); + let platform = Arc::new(SimPlatformStore::no_faults(297)); + let csdl = parse_csdl(CSDL_XML).expect("CSDL parse"); + let mut registry = SpecRegistry::new(); + registry.register_tenant( + tenant.as_str(), + csdl, + CSDL_XML.to_string(), + &[("Doc", DOC_IOA)], + ); + let mut server = ServerState::from_registry( + ActorSystem::new("arn238-serialized-spec-publications"), + registry, + ); + server.set_storage_stack(StorageStack::from_sim(events, Some(Arc::clone(&platform)))); + + let mut first_guard = server + .begin_spec_publication(&tenant) + .await + .expect("acquire first publication"); + let second_error = match server.begin_spec_publication(&tenant).await { + Ok(_) => panic!("second publication crossed the first tenant guard"), + Err(error) => error, + }; + assert!(second_error.contains("runtime generation is busy")); + assert!( + PlatformStore::load_specs(platform.as_ref()) + .await + .expect("load pre-publication specs") + .is_empty(), + "a waiting publication must not mutate durable specs" + ); + + publish_doc_generation( + &server, + platform.as_ref(), + &mut first_guard, + &tenant, + DOC_IOA, + ) + .await; + drop(first_guard); + let mut second_guard = server + .begin_spec_publication(&tenant) + .await + .expect("retry second publication after first cutover"); + publish_doc_generation( + &server, + platform.as_ref(), + &mut second_guard, + &tenant, + DOC_IOA_WITHOUT_KEYS, + ) + .await; + + let durable = PlatformStore::load_specs(platform.as_ref()) + .await + .expect("load final durable specs"); + assert_eq!(durable.len(), 1); + assert_eq!(durable[0].ioa_source, DOC_IOA_WITHOUT_KEYS); + let live_table = server + .registry + .read() + .expect("registry lock") + .get_table(&tenant, "Doc") + .expect("live Doc table"); + assert!(live_table.keys.is_empty(), "live registry must finish on B"); + assert_eq!( + server + .storage_stack + .as_ref() + .expect("storage") + .events + .key_index_backfilled_types(tenant.as_str()) + .await + .expect("final readiness"), + vec![( + "Doc".to_string(), + declared_key_set_signature( + &TransitionTable::from_ioa_source(DOC_IOA_WITHOUT_KEYS).keys + ) + )], + "active ownership readiness must finish on B" + ); +} + +/// Tenant requests hold the read side of the publication barrier for their +/// entire handler. A publication fails with a bounded retry signal while an +/// entered request owns the generation, later requests fail admission while +/// the writer owns the generation, and an abandoned +/// publication remains explicitly gated until its exact retry completes. +#[tokio::test] +async fn tenant_request_generation_barrier_is_bounded_and_fail_closed() { + let (_guard, _clock, _ids) = install_deterministic_context(300); + let tenant = TenantId::default(); + let mut registry = SpecRegistry::new(); + registry.register_tenant( + tenant.as_str(), + parse_csdl(CSDL_XML).expect("CSDL parse"), + CSDL_XML.to_string(), + &[("Doc", DOC_IOA)], + ); + let server = ServerState::from_registry( + ActorSystem::new("arn238-tenant-generation-barrier"), + registry, + ); + let request_generation = server.begin_tenant_request(&tenant).await; + let busy_error = match server.begin_spec_publication(&tenant).await { + Ok(_) => panic!("publication crossed an entered tenant request"), + Err(error) => error, + }; + assert!(busy_error.contains("runtime generation is busy")); + drop(request_generation); + + let mut interrupted = server + .begin_spec_publication(&tenant) + .await + .expect("acquire publication after request exits"); + let intent = doc_publication_intent(DOC_IOA); + server + .arm_spec_publication(&mut interrupted, &tenant, &intent) + .expect("arm interrupted publication"); + assert!( + server.try_begin_tenant_request(&tenant).await.is_none(), + "request admission must be nonblocking while publication owns the generation" + ); + drop(interrupted); + + let sticky_request = server + .try_begin_tenant_request(&tenant) + .await + .expect("writer guard unwound"); + assert!( + server.spec_publication_gated(&tenant), + "an outcome-ambiguous publication must still return retryable service-unavailable" + ); + drop(sticky_request); + + let mut different = server + .begin_spec_publication(&tenant) + .await + .expect("acquire different-intent attempt"); + let different_error = server + .arm_spec_publication( + &mut different, + &tenant, + &doc_publication_intent(DOC_IOA_WITHOUT_KEYS), + ) + .expect_err("a different generation cannot discharge sticky publication debt"); + assert!(different_error.contains("retry its exact runtime generation")); + drop(different); + assert!(server.spec_publication_gated(&tenant)); + + let mut retry = server + .begin_spec_publication(&tenant) + .await + .expect("acquire exact retry"); + server + .arm_spec_publication(&mut retry, &tenant, &intent) + .expect("arm exact retry"); + let mut cutover = server + .prepare_key_index_contracts_for_spec_activation(&retry, &tenant, &[("Doc", DOC_IOA)]) + .await + .expect("prepare no-store cutover"); + server + .finish_key_index_contract_activation(&mut retry, &tenant, &mut cutover) + .await + .expect("finish no-store cutover"); + server + .complete_spec_publication_retry(&mut retry, &tenant) + .expect("complete exact retry"); + assert!(!server.spec_publication_gated(&tenant)); + drop(retry); + assert!(server.try_begin_tenant_request(&tenant).await.is_some()); +} + +/// Once durable publication starts, every error is outcome-ambiguous. A +/// committed generation whose activation fails, a definitely pre-commit retry +/// failure, and a commit whose acknowledgement is lost must all leave actor +/// spawn and writes fail-closed until one serialized retry finishes the exact +/// durable generation. +#[tokio::test] +async fn persist_first_activation_failures_remain_gated_until_successful_retry() { + use temper_server::platform_store::{PlatformStore, SimPlatformStore}; + + let (_guard, _clock, _ids) = install_deterministic_context(299); + let tenant = TenantId::default(); + let sim = SimEventStore::no_faults(299); + let platform = Arc::new(SimPlatformStore::no_faults(299)); + let mut registry = SpecRegistry::new(); + registry.register_tenant( + tenant.as_str(), + parse_csdl(CSDL_XML).expect("CSDL parse"), + CSDL_XML.to_string(), + &[("Doc", DOC_IOA)], + ); + let mut server = ServerState::from_registry( + ActorSystem::new("arn238-persist-first-activation-failure"), + registry, + ); + server.set_storage_stack(StorageStack::from_sim( + sim.clone(), + Some(Arc::clone(&platform)), + )); + + let mut initial = server + .begin_spec_publication(&tenant) + .await + .expect("acquire initial publication"); + publish_doc_generation(&server, platform.as_ref(), &mut initial, &tenant, DOC_IOA).await; + drop(initial); + assert!( + server + .get_or_spawn_tenant_actor(&tenant, "Doc", "before-failure") + .is_some(), + "the completed initial generation must accept actor spawn" + ); + + let mut failed_activation = server + .begin_spec_publication(&tenant) + .await + .expect("acquire changed publication"); + server + .arm_spec_publication( + &mut failed_activation, + &tenant, + &doc_publication_intent(DOC_IOA_WITHOUT_KEYS), + ) + .expect("arm changed generation"); + persist_doc_generation(platform.as_ref(), &tenant, DOC_IOA_WITHOUT_KEYS) + .await + .expect("commit changed durable generation"); + sim.fail_next_key_activations(1); + let activation_error = server + .prepare_key_index_contracts_for_spec_activation( + &failed_activation, + &tenant, + &[("Doc", DOC_IOA_WITHOUT_KEYS)], + ) + .await + .expect_err("injected key-contract activation must fail"); + assert!(activation_error.contains("injected key activation failure")); + drop(failed_activation); + assert!( + server + .get_or_spawn_tenant_actor(&tenant, "Doc", "after-activation-failure") + .is_none(), + "post-commit activation failure must keep actor spawn gated" + ); + + let mut unrelated_retry = server + .begin_spec_publication(&tenant) + .await + .expect("acquire unrelated retry"); + let unrelated_error = server + .arm_spec_publication( + &mut unrelated_retry, + &tenant, + &unrelated_task_publication_intent(), + ) + .expect_err("an unrelated partial publication cannot inherit Doc's gate"); + assert!(unrelated_error.contains("retry its exact runtime generation")); + drop(unrelated_retry); + assert!( + server + .get_or_spawn_tenant_actor(&tenant, "Doc", "after-unrelated-retry") + .is_none(), + "an unrelated Task merge must not discharge the unresolved Doc generation" + ); + let durable_after_unrelated = PlatformStore::load_specs(platform.as_ref()) + .await + .expect("load durable generation after rejected unrelated retry"); + assert_eq!(durable_after_unrelated.len(), 1); + assert_eq!(durable_after_unrelated[0].ioa_source, DOC_IOA_WITHOUT_KEYS); + + let mut precommit_retry = server + .begin_spec_publication(&tenant) + .await + .expect("acquire pre-commit retry"); + server + .arm_spec_publication( + &mut precommit_retry, + &tenant, + &doc_publication_intent(DOC_IOA_WITHOUT_KEYS), + ) + .expect("arm pre-commit retry"); + platform.fail_next_spec_publications(1); + persist_doc_generation(platform.as_ref(), &tenant, DOC_IOA_WITHOUT_KEYS) + .await + .expect_err("inject pre-commit retry failure"); + drop(precommit_retry); + assert!( + server + .get_or_spawn_tenant_actor(&tenant, "Doc", "after-precommit-failure") + .is_none(), + "a failed retry cannot clear an inherited unresolved gate" + ); + + let mut ambiguous_retry = server + .begin_spec_publication(&tenant) + .await + .expect("acquire ambiguous retry"); + server + .arm_spec_publication( + &mut ambiguous_retry, + &tenant, + &doc_publication_intent(DOC_IOA_WITHOUT_KEYS), + ) + .expect("arm ambiguous retry"); + platform.fail_next_spec_publications_after_commit(1); + let ambiguous_error = persist_doc_generation(platform.as_ref(), &tenant, DOC_IOA_WITHOUT_KEYS) + .await + .expect_err("inject lost commit acknowledgement"); + assert!(ambiguous_error.contains("post-commit publication failure")); + drop(ambiguous_retry); + assert!( + server + .get_or_spawn_tenant_actor(&tenant, "Doc", "after-ambiguous-failure") + .is_none(), + "an outcome-ambiguous commit must keep actor spawn gated" + ); + let durable = PlatformStore::load_specs(platform.as_ref()) + .await + .expect("load committed generation after ambiguous error"); + assert_eq!(durable.len(), 1); + assert_eq!(durable[0].ioa_source, DOC_IOA_WITHOUT_KEYS); + assert!( + !server + .registry + .read() + .expect("registry lock") + .get_table(&tenant, "Doc") + .expect("old live Doc table") + .keys + .is_empty(), + "the live registry must remain on the old generation while gated" + ); + + let mut recovered = server + .begin_spec_publication(&tenant) + .await + .expect("acquire successful retry"); + publish_doc_generation( + &server, + platform.as_ref(), + &mut recovered, + &tenant, + DOC_IOA_WITHOUT_KEYS, + ) + .await; + drop(recovered); + assert!( + server + .registry + .read() + .expect("registry lock") + .get_table(&tenant, "Doc") + .expect("recovered live Doc table") + .keys + .is_empty(), + "successful retry must cut over the exact durable generation" + ); + assert!( + server + .get_or_spawn_tenant_actor(&tenant, "Doc", "after-recovery") + .is_some(), + "successful activation and readiness publication must reopen actor spawn" + ); +} + +/// Duplicate rows must be rejected before a publication mutates any durable +/// generation. Otherwise persistence can commit a last-wins row that runtime +/// activation rejects as structurally invalid. +#[tokio::test] +async fn duplicate_spec_publication_is_rejected_without_mutating_durable_generation() { + use temper_server::platform_store::{ + PlatformStore, SimPlatformStore, SpecPublication, SpecPublicationMode, + TenantConstraintsPublication, TenantPolicyPublication, + }; + + let platform = SimPlatformStore::no_faults(298); + let original_hash = temper_store_turso::spec_content_hash(DOC_IOA); + PlatformStore::publish_specs( + &platform, + "default", + &[SpecPublication { + entity_type: "Doc", + ioa_source: DOC_IOA, + csdl_xml: CSDL_XML, + content_hash: &original_hash, + }], + SpecPublicationMode::Replace, + TenantConstraintsPublication::Preserve, + TenantPolicyPublication::Preserve, + None, + None, + &[], + ) + .await + .expect("seed original generation"); + let replacement_hash = temper_store_turso::spec_content_hash(DOC_IOA_WITHOUT_KEYS); + let error = PlatformStore::publish_specs( + &platform, + "default", + &[ + SpecPublication { + entity_type: "Doc", + ioa_source: DOC_IOA, + csdl_xml: CSDL_XML, + content_hash: &original_hash, + }, + SpecPublication { + entity_type: "Doc", + ioa_source: DOC_IOA_WITHOUT_KEYS, + csdl_xml: CSDL_XML, + content_hash: &replacement_hash, + }, + ], + SpecPublicationMode::Replace, + TenantConstraintsPublication::Preserve, + TenantPolicyPublication::Preserve, + None, + None, + &[], + ) + .await + .expect_err("duplicate publication must fail preflight"); + assert!(error.contains("duplicate entity type Doc")); + let durable = PlatformStore::load_specs(&platform) + .await + .expect("load durable generation after rejection"); + assert_eq!(durable.len(), 1); + assert_eq!(durable[0].ioa_source, DOC_IOA); +} diff --git a/crates/temper-server/tests/key_ownership_write_surfaces/tombstone_snapshot_recovery.rs b/crates/temper-server/tests/key_ownership_write_surfaces/tombstone_snapshot_recovery.rs new file mode 100644 index 000000000..7bb6ff8b3 --- /dev/null +++ b/crates/temper-server/tests/key_ownership_write_surfaces/tombstone_snapshot_recovery.rs @@ -0,0 +1,356 @@ +//! Recovery regression for a terminal tombstone hidden behind a newer stale snapshot. + +use super::*; +use temper_runtime::persistence::{EventMetadata, EventStore, PersistenceEnvelope}; +use temper_runtime::scheduler::sim_now; +use temper_server::entity_actor::types::EntityEvent; +use temper_store_sim::SimFaultConfig; + +const PERSISTENCE_ID: &str = "default:Doc:stale-snapshot"; + +/// A first journal generation can replace a snapshot-only migration generation +/// without advancing the aggregate sequence. Actor restart must follow the +/// journal just like key-authority reads and backfill do. +#[tokio::test] +async fn actor_recovery_prefers_equal_sequence_journal_generation_over_snapshot() { + let (_guard, _clock, _ids) = install_deterministic_context(276); + let sim = SimEventStore::no_faults(276); + let events = BoxedEventStore::new(sim); + let persistence_id = "default:Doc:equal-sequence-source"; + let snapshot = serde_json::json!({ + "entity_type": "Doc", + "entity_id": "equal-sequence-source", + "status": "Ready", + "item_count": 0, + "fields": { + "Id": "equal-sequence-source", + "Status": "Ready", + "WorkspaceId": "ws", + "Path": "/snapshot-generation" + } + }); + events + .save_snapshot( + persistence_id, + 1, + &serde_json::to_vec(&snapshot).expect("serialize snapshot-only generation"), + ) + .await + .expect("seed snapshot-only generation"); + + let timestamp = sim_now(); + events + .append( + persistence_id, + 0, + &[PersistenceEnvelope { + sequence_nr: 1, + event_type: "Temper.Internal.FieldUpdate.v1".to_string(), + payload: serde_json::json!({ + "schema": "temper.field-update.v1", + "fields": { + "Path": "/journal-generation", + "JournalOnly": "must-survive" + }, + "replace": false, + "idempotency_key": "equal-sequence-source" + }), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: persistence_id.to_string(), + }, + }], + ) + .await + .expect("replace snapshot-only source with first journal generation"); + + let table = Arc::new(RwLock::new(TransitionTable::from_ioa_source(DOC_IOA))); + let restarted_system = ActorSystem::new("arn238-equal-sequence-source-restart"); + let restarted = restarted_system.spawn( + EntityActor::with_persistence( + "Doc", + "equal-sequence-source", + table, + serde_json::json!({}), + events, + BackendLabel::Sim, + ) + .with_tenant("default"), + "equal-sequence-source", + ); + let recovered = state(&restarted).await.state; + assert_eq!( + recovered.fields["Path"], "/journal-generation", + "the first journal generation, not the equal-sequence snapshot, owns current state" + ); + assert_eq!(recovered.fields["JournalOnly"], "must-survive"); + assert_eq!( + recovered.status, "New", + "the journal generation owns lifecycle rather than the snapshot status" + ); + assert_eq!(recovered.fields["Status"], "New"); + assert_eq!(recovered.sequence_nr, 1); +} + +async fn persist_tombstone_behind_stale_live_snapshot( + events: &BoxedEventStore, + table: &Arc>, +) -> u64 { + let live_system = ActorSystem::new("arn238-tombstone-snapshot-live"); + let live_actor = live_system.spawn( + EntityActor::with_persistence( + "Doc", + "stale-snapshot", + table.clone(), + serde_json::json!({}), + events.clone(), + BackendLabel::Sim, + ) + .with_tenant("default"), + "stale-snapshot", + ); + assert!( + action( + &live_actor, + "Create", + serde_json::json!({"WorkspaceId": "ws", "Path": "/deleted"}), + ) + .await + .success + ); + let mut stale_live_snapshot = state(&live_actor).await.state; + let live_sequence = stale_live_snapshot.sequence_nr; + let tombstone_sequence = live_sequence + 1; + let stale_snapshot_sequence = tombstone_sequence + 1; + drop(live_actor); + drop(live_system); + + let deleted_at = sim_now(); + events + .append( + PERSISTENCE_ID, + live_sequence, + &[PersistenceEnvelope { + sequence_nr: tombstone_sequence, + event_type: "Delete".to_string(), + payload: serde_json::to_value(EntityEvent { + action: "Delete".to_string(), + from_status: "Ready".to_string(), + to_status: "Deleted".to_string(), + timestamp: deleted_at, + params: serde_json::json!({}), + idempotency_key: None, + }) + .expect("serialize tombstone"), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: deleted_at, + actor_id: PERSISTENCE_ID.to_string(), + }, + }], + ) + .await + .expect("append action-named tombstone"); + + stale_live_snapshot.sequence_nr = stale_snapshot_sequence; + stale_live_snapshot.last_snapshot_sequence_nr = stale_snapshot_sequence; + stale_live_snapshot.events_since_snapshot = 0; + events + .save_snapshot( + PERSISTENCE_ID, + stale_snapshot_sequence, + &serde_json::to_vec(&stale_live_snapshot).expect("serialize stale live snapshot"), + ) + .await + .expect("persist newer stale live snapshot"); + + tombstone_sequence +} + +/// A tombstone is terminal durable history. A newer stale snapshot must not let a +/// restarted actor reappear live, even when the domain action is named `Delete` +/// rather than the legacy canonical `Deleted` event. +#[tokio::test] +async fn actor_recovery_replays_older_tombstone_before_newer_live_snapshot() { + let (_guard, _clock, _ids) = install_deterministic_context(246); + let sim = SimEventStore::no_faults(246); + let events = BoxedEventStore::new(sim.clone()); + let table = Arc::new(RwLock::new(TransitionTable::from_ioa_source(DOC_IOA))); + let tombstone_sequence = persist_tombstone_behind_stale_live_snapshot(&events, &table).await; + + let restarted_system = ActorSystem::new("arn238-tombstone-snapshot-restart"); + let restarted = restarted_system.spawn( + EntityActor::with_persistence( + "Doc", + "stale-snapshot", + table, + serde_json::json!({}), + events, + BackendLabel::Sim, + ) + .with_tenant("default"), + "stale-snapshot", + ); + let recovered = state(&restarted).await.state; + assert_eq!(recovered.status, "Deleted"); + assert_eq!( + recovered.sequence_nr, tombstone_sequence, + "the terminal journal boundary, not the stale snapshot, is authoritative" + ); + assert_eq!( + recovered.last_snapshot_sequence_nr, 0, + "a rejected snapshot is not an accepted actor snapshot boundary" + ); +} + +/// A fault-truncated replay must fail closed at the exact tombstone boundary; +/// after the transient fault clears, a fresh recovery reaches `Deleted`. +#[tokio::test] +async fn actor_recovery_never_accepts_live_state_when_tombstone_replay_truncates() { + let (_guard, _clock, _ids) = install_deterministic_context(247); + let sim = SimEventStore::no_faults(247); + let events = BoxedEventStore::new(sim.clone()); + let table = Arc::new(RwLock::new(TransitionTable::from_ioa_source(DOC_IOA))); + persist_tombstone_behind_stale_live_snapshot(&events, &table).await; + sim.restore_faults(SimFaultConfig { + write_failure_prob: 0.0, + concurrency_violation_prob: 0.0, + read_truncation_prob: 1.0, + snapshot_failure_prob: 0.0, + }); + + let truncated_system = ActorSystem::new("arn238-tombstone-truncated-replay"); + let truncated = truncated_system.spawn( + EntityActor::with_persistence( + "Doc", + "stale-snapshot", + table.clone(), + serde_json::json!({}), + events.clone(), + BackendLabel::Sim, + ) + .with_tenant("default"), + "stale-snapshot-truncated", + ); + let truncated_result = truncated + .ask::(EntityMsg::GetState, Duration::from_secs(1)) + .await; + assert!( + truncated_result.is_err(), + "recovery must not serve a live prefix when replay stops before a known tombstone" + ); + drop(truncated); + drop(truncated_system); + + sim.disable_faults(); + let recovered_system = ActorSystem::new("arn238-tombstone-replay-retry"); + let recovered = recovered_system.spawn( + EntityActor::with_persistence( + "Doc", + "stale-snapshot", + table, + serde_json::json!({}), + events, + BackendLabel::Sim, + ) + .with_tenant("default"), + "stale-snapshot-retry", + ); + assert_eq!(state(&recovered).await.state.status, "Deleted"); +} + +/// The first terminal journal event is irreversible even if a legacy writer +/// left a later non-terminal suffix. An already-indexed entity must be removed +/// from memory without treating the final envelope as renewed liveness. +#[tokio::test] +async fn ensure_loaded_rejects_indexed_entity_with_suffix_after_tombstone() { + let (_guard, _clock, _ids) = install_deterministic_context(301); + let tenant = TenantId::default(); + let sim = SimEventStore::no_faults(301); + let mut registry = SpecRegistry::new(); + registry.register_tenant( + tenant.as_str(), + parse_csdl(CSDL_XML).expect("CSDL parse"), + CSDL_XML.to_string(), + &[("Doc", DOC_IOA)], + ); + let mut server = ServerState::from_registry( + ActorSystem::new("arn238-terminal-suffix-ensure-loaded"), + registry, + ); + server.set_storage_stack(StorageStack::from_sim(sim.clone(), None)); + let created = server + .get_or_create_tenant_entity( + &tenant, + "Doc", + "terminal-suffix", + serde_json::json!({"WorkspaceId": "ws", "Path": "/terminal"}), + ) + .await + .expect("create indexed entity"); + assert!(server.entity_exists(&tenant, "Doc", "terminal-suffix")); + + let persistence_id = "default:Doc:terminal-suffix"; + let timestamp = sim_now(); + sim.append( + persistence_id, + created.state.sequence_nr, + &[ + PersistenceEnvelope { + sequence_nr: 0, + event_type: "Delete".to_string(), + payload: serde_json::to_value(EntityEvent { + action: "Delete".to_string(), + from_status: "Ready".to_string(), + to_status: "Deleted".to_string(), + timestamp, + params: serde_json::json!({}), + idempotency_key: None, + }) + .expect("serialize terminal event"), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: persistence_id.to_string(), + }, + }, + PersistenceEnvelope { + sequence_nr: 0, + event_type: "LegacySuffix".to_string(), + payload: serde_json::to_value(EntityEvent { + action: "LegacySuffix".to_string(), + from_status: "Deleted".to_string(), + to_status: "Ready".to_string(), + timestamp, + params: serde_json::json!({}), + idempotency_key: None, + }) + .expect("serialize legacy suffix"), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: persistence_id.to_string(), + }, + }, + ], + ) + .await + .expect("seed legacy suffix after terminal event"); + + assert!( + !server + .ensure_entity_loaded(&tenant, "Doc", "terminal-suffix") + .await, + "the first terminal boundary must override a later legacy suffix" + ); + assert!(!server.entity_exists(&tenant, "Doc", "terminal-suffix")); +} diff --git a/crates/temper-server/tests/passivation_respawn.rs b/crates/temper-server/tests/passivation_respawn.rs index dc5bacb08..0c368d5bf 100644 --- a/crates/temper-server/tests/passivation_respawn.rs +++ b/crates/temper-server/tests/passivation_respawn.rs @@ -2,11 +2,184 @@ mod common; -use temper_runtime::persistence::EventStore; -use temper_runtime::scheduler::{install_deterministic_context, sim_now}; +use temper_runtime::persistence::{EventMetadata, EventStore, PersistenceEnvelope}; +use temper_runtime::scheduler::{install_deterministic_context, sim_now, sim_uuid}; use temper_runtime::tenant::TenantId; use temper_store_sim::SimEventStore; +#[tokio::test] +async fn passivation_does_not_overwrite_a_same_sequence_snapshot_rewrite() { + let seed = 281; + let (_guard, _clock, _id_gen) = install_deterministic_context(seed); + let sim_store = SimEventStore::no_faults(seed); + let state = common::build_default_state_with_store(sim_store.clone(), "passivation-cas"); + let tenant = TenantId::default(); + let entity_id = "same-sequence-passivation"; + let actor_key = format!("{tenant}:Order:{entity_id}"); + let legacy_snapshot = |marker: &str| { + serde_json::to_vec(&serde_json::json!({ + "entity_type": "Order", + "entity_id": entity_id, + "status": "Draft", + "item_count": 0, + "fields": { + "Id": entity_id, + "Status": "Draft", + "Marker": marker + } + })) + .expect("serialize legacy snapshot") + }; + let captured_snapshot = legacy_snapshot("captured"); + sim_store + .save_snapshot(&actor_key, 1, &captured_snapshot) + .await + .expect("seed captured snapshot"); + let timestamp = sim_now(); + sim_store + .append( + &actor_key, + 0, + &[PersistenceEnvelope { + sequence_nr: 1, + event_type: "Temper.Internal.FieldUpdate.v1".to_string(), + payload: serde_json::json!({ + "schema": "temper.field-update.v1", + "fields": {"JournalMarker": "journal"}, + "replace": false, + "idempotency_key": "passivation-cas-seed" + }), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: actor_key.clone(), + }, + }], + ) + .await + .expect("seed equal-sequence journal"); + + let hydrated = state + .get_tenant_entity_state(&tenant, "Order", entity_id) + .await + .expect("hydrate captured snapshot"); + assert_eq!(hydrated.state.fields["Marker"], "captured"); + + let replacement_snapshot = legacy_snapshot("replacement"); + sim_store + .save_snapshot(&actor_key, 1, &replacement_snapshot) + .await + .expect("install concurrent same-sequence replacement"); + state + .last_accessed + .write() + .expect("last-accessed lock") + .insert( + actor_key.clone(), + sim_now() - chrono::Duration::seconds(600), + ); + + state.passivate_idle_actors().await; + + assert_eq!( + sim_store + .load_snapshot(&actor_key) + .await + .expect("load snapshot after passivation"), + Some((1, replacement_snapshot)), + "passivation must not replace a newer same-sequence snapshot source with stale actor state" + ); +} + +#[tokio::test] +async fn passivated_snapshot_only_actor_does_not_claim_journal_provenance() { + let seed = 278; + let (_guard, _clock, _id_gen) = install_deterministic_context(seed); + let sim_store = SimEventStore::no_faults(seed); + let state = common::build_default_state_with_store(sim_store.clone(), "passivation-source"); + let tenant = TenantId::default(); + let entity_id = "snapshot-only-passivation"; + let actor_key = format!("{tenant}:Order:{entity_id}"); + let snapshot = serde_json::json!({ + "entity_type": "Order", + "entity_id": entity_id, + "status": "Draft", + "item_count": 0, + "fields": { + "Id": entity_id, + "Status": "Draft", + "Marker": "snapshot-generation" + } + }); + sim_store + .save_snapshot( + &actor_key, + 1, + &serde_json::to_vec(&snapshot).expect("serialize snapshot-only generation"), + ) + .await + .expect("seed snapshot-only generation"); + + let hydrated = state + .get_tenant_entity_state(&tenant, "Order", entity_id) + .await + .expect("hydrate snapshot-only actor"); + assert_eq!(hydrated.state.fields["Marker"], "snapshot-generation"); + { + let mut last_accessed = state.last_accessed.write().expect("last-accessed lock"); + last_accessed.insert( + actor_key.clone(), + sim_now() - chrono::Duration::seconds(600), + ); + } + state.passivate_idle_actors().await; + assert!( + !state + .actor_registry + .read() + .expect("actor registry lock") + .contains_key(&actor_key), + "fixture must passivate the snapshot-only actor" + ); + + let timestamp = sim_now(); + sim_store + .append( + &actor_key, + 0, + &[PersistenceEnvelope { + sequence_nr: 1, + event_type: "Temper.Internal.FieldUpdate.v1".to_string(), + payload: serde_json::json!({ + "schema": "temper.field-update.v1", + "fields": {"Marker": "journal-generation"}, + "replace": false, + "idempotency_key": "passivation-source-replacement" + }), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: actor_key.clone(), + }, + }], + ) + .await + .expect("replace snapshot-only source with first journal generation"); + + let recovered = state + .get_tenant_entity_state(&tenant, "Order", entity_id) + .await + .expect("respawn after equal-sequence journal replacement"); + assert_eq!( + recovered.state.fields["Marker"], "journal-generation", + "passivation must not forge journal provenance for snapshot-only state" + ); +} + #[tokio::test] async fn passivated_actor_respawns_with_correct_state() { let seed = 42; diff --git a/crates/temper-server/tests/single_entity_projection_fence.rs b/crates/temper-server/tests/single_entity_projection_fence.rs new file mode 100644 index 000000000..608fffa69 --- /dev/null +++ b/crates/temper-server/tests/single_entity_projection_fence.rs @@ -0,0 +1,253 @@ +//! Single-entity OData reads must close against durable entity sources. + +mod common; + +use std::sync::{Arc, Mutex}; + +use async_trait::async_trait; +use axum::body::Body; +use axum::http::{Request, StatusCode}; +use temper_runtime::ActorSystem; +use temper_runtime::persistence::{EventStore, PersistenceError}; +use temper_runtime::tenant::TenantId; +use temper_server::registry::SpecRegistry; +use temper_server::storage::{ + BackendLabel, BoxedEventStore, EntityCatalogRow, QueryPlaneStore, QueryProjectionFieldsRow, +}; +use temper_server::{ServerState, StorageStack, build_router}; +use temper_spec::csdl::parse_csdl; +use temper_store_sim::SimEventStore; +use tower::ServiceExt; + +type SnapshotWrite = (SimEventStore, String, u64, Vec); + +struct StaleCatalog { + row: EntityCatalogRow, + snapshot_on_load: Mutex>, +} + +#[async_trait] +impl QueryPlaneStore for StaleCatalog { + async fn upsert_projection( + &self, + _tenant: &str, + _entity_type: &str, + _entity_id: &str, + _status: &str, + _fields: &serde_json::Value, + _state: &serde_json::Value, + _sequence_nr: u64, + ) -> Result<(), PersistenceError> { + Ok(()) + } + + async fn remove_projection( + &self, + _tenant: &str, + _entity_type: &str, + _entity_id: &str, + ) -> Result<(), PersistenceError> { + Ok(()) + } + + async fn query_field_index( + &self, + _tenant: &str, + _entity_type: &str, + _where_clause: &str, + _params: Vec, + ) -> Result>, PersistenceError> { + Ok(None) + } + + async fn load_projection_fields_many( + &self, + _tenant: &str, + _entity_type: &str, + _entity_ids: &[String], + _field_names: &[&str], + ) -> Result>, PersistenceError> { + Ok(None) + } + + async fn load_entity_catalog_rows( + &self, + _tenant: &str, + entity_type: &str, + entity_ids: &[String], + ) -> Result>, PersistenceError> { + let snapshot_write = self + .snapshot_on_load + .lock() + .expect("snapshot injection lock poisoned") + .take(); + if let Some((store, persistence_id, sequence_nr, snapshot)) = snapshot_write { + EventStore::save_snapshot(&store, &persistence_id, sequence_nr, &snapshot).await?; + } + if entity_type == "Order" && entity_ids.contains(&self.row.entity_id) { + return Ok(Some(vec![self.row.clone()])); + } + Ok(Some(Vec::new())) + } + + async fn projected_entity_counts_by_tenant( + &self, + ) -> Result>, PersistenceError> { + Ok(None) + } +} + +fn build_stale_catalog_state( + tenant: &TenantId, + store: SimEventStore, + stale_catalog: Arc, + system_name: &str, +) -> ServerState { + let mut registry = SpecRegistry::new(); + registry.register_tenant( + tenant.as_str(), + parse_csdl(common::CSDL_XML).expect("parse CSDL"), + common::CSDL_XML.to_string(), + &[("Order", common::ORDER_IOA)], + ); + let mut state = ServerState::from_registry(ActorSystem::new(system_name), registry); + state.set_storage_stack(StorageStack::new( + BackendLabel::Sim, + BoxedEventStore::new(store), + None, + None, + None, + None, + Some(stale_catalog as Arc), + None, + None, + None, + )); + state +} + +#[tokio::test] +async fn direct_key_get_does_not_resurrect_durable_tombstone_from_stale_catalog() { + let tenant = TenantId::default(); + let entity_id = "ord-direct-tombstone"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let store = SimEventStore::no_faults(9_238); + let tombstone = serde_json::json!({ + "entity_type": "Order", + "entity_id": entity_id, + "status": "Deleted", + "item_count": 0, + "fields": { + "Id": entity_id, + "Status": "Deleted", + "WorkspaceId": null, + "Path": null, + }, + "sequence_nr": 5, + }); + EventStore::save_snapshot( + &store, + &persistence_id, + 5, + &serde_json::to_vec(&tombstone).expect("serialize durable tombstone"), + ) + .await + .expect("persist durable tombstone"); + + let stale_fields = serde_json::json!({ + "Id": entity_id, + "Status": "Created", + "WorkspaceId": "ws-stale", + "Path": "/stale", + }); + let stale_catalog = Arc::new(StaleCatalog { + row: EntityCatalogRow { + entity_id: entity_id.to_string(), + status: "Created".to_string(), + fields: stale_fields.clone(), + state: Some(serde_json::json!({ + "entity_type": "Order", + "entity_id": entity_id, + "status": "Created", + "fields": stale_fields, + "sequence_nr": 4, + })), + sequence_nr: 4, + }, + snapshot_on_load: Mutex::new(None), + }); + let state = + build_stale_catalog_state(&tenant, store, stale_catalog, "direct-key-projection-fence"); + + let response = build_router(state) + .oneshot( + Request::get(format!("/tdata/Orders('{entity_id}')")) + .body(Body::empty()) + .expect("build request"), + ) + .await + .expect("read response"); + + assert_eq!(response.status(), StatusCode::NOT_FOUND); +} + +#[tokio::test] +async fn direct_key_get_rechecks_durable_absence_after_catalog_materialization() { + let tenant = TenantId::default(); + let entity_id = "ord-direct-source-transition"; + let persistence_id = format!("{tenant}:Order:{entity_id}"); + let store = SimEventStore::no_faults(9_239); + let tombstone = serde_json::to_vec(&serde_json::json!({ + "entity_type": "Order", + "entity_id": entity_id, + "status": "Deleted", + "item_count": 0, + "fields": { + "Id": entity_id, + "Status": "Deleted", + "WorkspaceId": null, + "Path": null, + }, + "sequence_nr": 5, + })) + .expect("serialize injected tombstone"); + let stale_fields = serde_json::json!({ + "Id": entity_id, + "Status": "Created", + "WorkspaceId": "ws-stale", + "Path": "/stale", + }); + let stale_catalog = Arc::new(StaleCatalog { + row: EntityCatalogRow { + entity_id: entity_id.to_string(), + status: "Created".to_string(), + fields: stale_fields.clone(), + state: Some(serde_json::json!({ + "entity_type": "Order", + "entity_id": entity_id, + "status": "Created", + "fields": stale_fields, + "sequence_nr": 4, + })), + sequence_nr: 4, + }, + snapshot_on_load: Mutex::new(Some((store.clone(), persistence_id, 5, tombstone))), + }); + let state = build_stale_catalog_state( + &tenant, + store, + stale_catalog, + "direct-key-source-transition", + ); + + let response = build_router(state) + .oneshot( + Request::get(format!("/tdata/Orders('{entity_id}')")) + .body(Body::empty()) + .expect("build request"), + ) + .await + .expect("read response"); + + assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE); +} diff --git a/crates/temper-server/tests/snapshot_only_materialization.rs b/crates/temper-server/tests/snapshot_only_materialization.rs new file mode 100644 index 000000000..63202c183 --- /dev/null +++ b/crates/temper-server/tests/snapshot_only_materialization.rs @@ -0,0 +1,316 @@ +//! Regression coverage for the first journal write after snapshot-only recovery. + +use std::collections::BTreeMap; +use std::sync::{Arc, RwLock}; +use std::time::Duration; + +use temper_jit::table::TransitionTable; +use temper_runtime::ActorSystem; +use temper_runtime::persistence::{EventMetadata, PersistenceEnvelope}; +use temper_runtime::scheduler::{install_deterministic_context, sim_now, sim_uuid}; +use temper_server::entity_actor::{EntityActor, EntityMsg, EntityResponse}; +use temper_server::storage::{BackendLabel, BoxedEventStore}; +use temper_store_sim::SimEventStore; + +const COUNTER_IOA: &str = r#" +[automaton] +name = "Counter" +states = ["Ready"] +initial = "Ready" + +[[state]] +name = "value" +type = "counter" +initial = "0" + +[[action]] +name = "Increment" +kind = "input" +from = ["Ready"] +to = "Ready" +effect = [{ type = "increment", var = "value" }] +"#; + +#[tokio::test] +async fn pre_fix_materialization_rebases_legacy_retry_coordinates() { + let (_guard, _clock, _ids) = install_deterministic_context(295); + let sim = SimEventStore::no_faults(295); + let events = BoxedEventStore::new(sim.clone()); + let persistence_id = "default:Counter:pre-fix-materialization"; + events + .append( + persistence_id, + 0, + &[PersistenceEnvelope { + sequence_nr: 0, + event_type: "Temper.Internal.StateMaterialization.v1".to_string(), + payload: serde_json::json!({ + "schema": "temper.state-materialization.v1", + "state": { + "entity_type": "Counter", + "entity_id": "pre-fix-materialization", + "status": "Ready", + "item_count": 0, + "counters": {"value": 10}, + "booleans": {}, + "lists": {}, + "fields": { + "Id": "pre-fix-materialization", + "Status": "Ready" + }, + "events": [], + "total_event_count": 10, + "events_since_snapshot": 0, + "last_snapshot_sequence_nr": 0, + "sequence_nr": 0, + "processed_idempotency_keys": { + "pre-fix-legacy-increment": 5 + } + } + }), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: sim_now(), + actor_id: persistence_id.to_string(), + }, + }], + ) + .await + .expect("seed pre-fix materialization payload"); + + let table = Arc::new(RwLock::new(TransitionTable::from_ioa_source(COUNTER_IOA))); + let system = ActorSystem::new("pre-fix-materialization-retry"); + let actor = system.spawn( + EntityActor::with_persistence( + "Counter", + "pre-fix-materialization", + table, + serde_json::json!({}), + events, + BackendLabel::Sim, + ) + .with_tenant("default"), + "pre-fix-materialization", + ); + let retry: EntityResponse = actor + .ask( + EntityMsg::Action { + name: "Increment".to_string(), + params: serde_json::json!({}), + cross_entity_booleans: BTreeMap::new(), + idempotency_key: Some("pre-fix-legacy-increment".to_string()), + }, + Duration::from_secs(5), + ) + .await + .expect("retry legacy token from pre-fix materialization"); + assert!( + retry.success, + "pre-fix materialized retry failed: {:?}", + retry.error + ); + assert_eq!(retry.state.counters.get("value"), Some(&10)); + assert_eq!( + sim.dump_journal(persistence_id).len(), + 1, + "legacy retry must not append" + ); +} + +#[tokio::test] +async fn first_journal_write_materializes_snapshot_only_state_for_restart() { + let (_guard, _clock, _ids) = install_deterministic_context(283); + let sim = SimEventStore::no_faults(283); + let events = BoxedEventStore::new(sim.clone()); + let persistence_id = "default:Counter:snapshot-only-counter"; + let snapshot = serde_json::to_vec(&serde_json::json!({ + "entity_type": "Counter", + "entity_id": "snapshot-only-counter", + "status": "Ready", + "item_count": 0, + "counters": {"value": 10}, + "booleans": {}, + "lists": {}, + "fields": { + "Id": "legacy-wrong-id", + "Status": "LegacyWrongStatus" + }, + "events": [], + "total_event_count": 10, + "events_since_snapshot": 0, + "last_snapshot_sequence_nr": 5, + "sequence_nr": 5, + "processed_idempotency_keys": { + "snapshot-only-legacy-increment": 5 + } + })) + .expect("serialize snapshot-only state"); + events + .save_snapshot(persistence_id, 5, &snapshot) + .await + .expect("seed snapshot-only generation"); + + let table = Arc::new(RwLock::new(TransitionTable::from_ioa_source(COUNTER_IOA))); + let first_system = ActorSystem::new("snapshot-only-materialization-first"); + let first = first_system.spawn( + EntityActor::with_persistence( + "Counter", + "snapshot-only-counter", + table.clone(), + serde_json::json!({}), + events.clone(), + BackendLabel::Sim, + ) + .with_tenant("default"), + "snapshot-only-counter-first", + ); + let updated: EntityResponse = first + .ask( + EntityMsg::Action { + name: "Increment".to_string(), + params: serde_json::json!({}), + cross_entity_booleans: BTreeMap::new(), + idempotency_key: Some("snapshot-only-increment".to_string()), + }, + Duration::from_secs(5), + ) + .await + .expect("increment snapshot-only actor"); + assert!(updated.success, "increment failed: {:?}", updated.error); + assert_eq!(updated.state.counters.get("value"), Some(&11)); + let journal = sim.dump_journal(persistence_id); + assert_eq!(journal.len(), 2); + assert_eq!( + journal[0].event_type, "Temper.Internal.StateMaterialization.v1", + "the first journal record must carry the complete snapshot-only baseline" + ); + assert_eq!(journal[1].event_type, "Increment"); + assert_eq!( + journal[0].payload["state"]["fields"]["Id"], "snapshot-only-counter", + "snapshot recovery must canonicalize field identity before journal materialization" + ); + assert_eq!( + journal[0].payload["state"]["fields"]["Status"], "Ready", + "snapshot recovery must canonicalize field status before journal materialization" + ); + let retry: EntityResponse = first + .ask( + EntityMsg::Action { + name: "Increment".to_string(), + params: serde_json::json!({}), + cross_entity_booleans: BTreeMap::new(), + idempotency_key: Some("snapshot-only-increment".to_string()), + }, + Duration::from_secs(5), + ) + .await + .expect("retry first journal action"); + assert!( + retry.success, + "immediate exact retry failed: {:?}", + retry.error + ); + assert_eq!(retry.state.counters.get("value"), Some(&11)); + assert_eq!( + sim.dump_journal(persistence_id).len(), + 2, + "immediate exact retry must not append" + ); + let legacy_retry: EntityResponse = first + .ask( + EntityMsg::Action { + name: "Increment".to_string(), + params: serde_json::json!({}), + cross_entity_booleans: BTreeMap::new(), + idempotency_key: Some("snapshot-only-legacy-increment".to_string()), + }, + Duration::from_secs(5), + ) + .await + .expect("retry materialized legacy token"); + assert!( + legacy_retry.success, + "materialized legacy token failed: {:?}", + legacy_retry.error + ); + assert_eq!(legacy_retry.state.counters.get("value"), Some(&11)); + assert_eq!( + sim.dump_journal(persistence_id).len(), + 2, + "materialized legacy token must not append" + ); + assert!( + events + .load_snapshot(persistence_id) + .await + .expect("load snapshot after materialization") + .is_none(), + "the fenced first journal append must atomically retire the migration snapshot" + ); + events + .save_snapshot(persistence_id, 50, &snapshot) + .await + .expect("attempt delayed migration snapshot write"); + assert!( + events + .load_snapshot(persistence_id) + .await + .expect("load snapshot after delayed migration writer") + .is_none(), + "a delayed ahead-of-journal writer must not recreate a retired migration snapshot" + ); + + let restart_system = ActorSystem::new("snapshot-only-materialization-restart"); + let restarted = restart_system.spawn( + EntityActor::with_persistence( + "Counter", + "snapshot-only-counter", + table, + serde_json::json!({}), + events, + BackendLabel::Sim, + ) + .with_tenant("default"), + "snapshot-only-counter-restarted", + ); + let recovered: EntityResponse = restarted + .ask(EntityMsg::GetState, Duration::from_secs(5)) + .await + .expect("recover actor after first journal write"); + + assert_eq!( + recovered.state.counters.get("value"), + Some(&11), + "restart must replay the first journal delta from the complete snapshot-only baseline" + ); + assert_eq!(recovered.state.fields["Id"], "snapshot-only-counter"); + assert_eq!(recovered.state.fields["Status"], "Ready"); + + for idempotency_key in ["snapshot-only-increment", "snapshot-only-legacy-increment"] { + let retry: EntityResponse = restarted + .ask( + EntityMsg::Action { + name: "Increment".to_string(), + params: serde_json::json!({}), + cross_entity_booleans: BTreeMap::new(), + idempotency_key: Some(idempotency_key.to_string()), + }, + Duration::from_secs(5), + ) + .await + .expect("retry materialized token after restart"); + assert!( + retry.success, + "post-restart retry failed for {idempotency_key}: {:?}", + retry.error + ); + assert_eq!(retry.state.counters.get("value"), Some(&11)); + assert_eq!( + sim.dump_journal(persistence_id).len(), + 2, + "post-restart retry must not append for {idempotency_key}" + ); + } +} diff --git a/crates/temper-server/tests/storage_stack.rs b/crates/temper-server/tests/storage_stack.rs index e49d338b7..0ac99d9a4 100644 --- a/crates/temper-server/tests/storage_stack.rs +++ b/crates/temper-server/tests/storage_stack.rs @@ -126,6 +126,7 @@ impl EventStore for RecordingEventStore { Ok(vec![PersistenceAppendResult { persistence_id: appends[0].persistence_id.clone(), sequence_nr: appends[0].events.len() as u64, + batch_already_applied: false, }]) } @@ -139,6 +140,19 @@ impl EventStore for RecordingEventStore { Ok(vec![test_envelope(1)]) } + async fn read_events_page( + &self, + persistence_id: &str, + from_sequence: u64, + through_sequence: u64, + limit: usize, + ) -> Result, PersistenceError> { + let mut events = EventStore::read_events(self, persistence_id, from_sequence).await?; + events.retain(|event| event.sequence_nr <= through_sequence); + events.truncate(limit); + Ok(events) + } + async fn save_snapshot( &self, persistence_id: &str, @@ -196,12 +210,18 @@ async fn boxed_event_store_delegates_through_object_safe_adapter() { persistence_id: "default:Ticket:t-1".to_string(), expected_sequence: 0, events: events.clone(), + key_rows: Vec::new(), + reconcile_keys: false, + key_set_signature: None, + snapshot_source: Default::default(), + batch_idempotency: None, }]) .await .expect("append batch through dyn adapter"), vec![PersistenceAppendResult { persistence_id: "default:Ticket:t-1".to_string(), sequence_nr: 2, + batch_already_applied: false, }] ); assert_eq!( @@ -239,6 +259,13 @@ async fn boxed_event_store_delegates_through_object_safe_adapter() { .expect("list by type through dyn adapter"), vec!["t-1".to_string()] ); + assert_eq!( + store + .list_entity_ids_for_key_reconciliation("default", "Ticket") + .await + .expect("list key-reconciliation owners through dyn adapter"), + vec!["t-1".to_string()] + ); assert_eq!( store .list_entity_ids_limited("default", Some("Ticket"), 1) diff --git a/crates/temper-store-postgres/Cargo.toml b/crates/temper-store-postgres/Cargo.toml index d7f6ea603..95dfa5715 100644 --- a/crates/temper-store-postgres/Cargo.toml +++ b/crates/temper-store-postgres/Cargo.toml @@ -18,3 +18,6 @@ tracing-opentelemetry = { workspace = true } uuid = { workspace = true } chrono = { workspace = true } sha2 = { workspace = true } + +[dev-dependencies] +futures = { workspace = true } diff --git a/crates/temper-store-postgres/migrations/0013_key_index_contract_revision.sql b/crates/temper-store-postgres/migrations/0013_key_index_contract_revision.sql new file mode 100644 index 000000000..bb28aee69 --- /dev/null +++ b/crates/temper-store-postgres/migrations/0013_key_index_contract_revision.sql @@ -0,0 +1,14 @@ +-- ADR-0192 / ARN-238: fence declared-key coverage against live key-contract changes. +-- +-- A durable write records the versioned key signature used to derive its exact rows. +-- Changing that signature advances `revision` and invalidates the coverage watermark. +-- Backfill publishes a new watermark only if this revision stayed unchanged while it +-- replayed the type, preventing remove/re-add and A -> B -> A ABA certification. +CREATE TABLE IF NOT EXISTS key_index_contract_state ( + tenant TEXT NOT NULL, + entity_type TEXT NOT NULL, + key_set TEXT NOT NULL, + revision BIGINT NOT NULL CHECK (revision >= 0), + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (tenant, entity_type) +); diff --git a/crates/temper-store-postgres/migrations/0014_query_projection_dirty.sql b/crates/temper-store-postgres/migrations/0014_query_projection_dirty.sql new file mode 100644 index 000000000..1fa6d043a --- /dev/null +++ b/crates/temper-store-postgres/migrations/0014_query_projection_dirty.sql @@ -0,0 +1,70 @@ +-- ADR-0192 / ARN-238: make derived query-projection lag explicit and durable. +-- +-- Journal and snapshot writers mark their entity dirty in the same stream-fenced +-- transaction as the source mutation. A source-fenced catalog/EAV repair clears +-- the row. Readers repair these bounded rows before trusting native projection +-- pages, so a delete/recreate or same-sequence snapshot rewrite cannot become a +-- silent false negative. +CREATE TABLE IF NOT EXISTS query_projection_dirty ( + tenant TEXT NOT NULL, + entity_type TEXT NOT NULL, + entity_id TEXT NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (tenant, entity_type, entity_id) +); + +CREATE INDEX IF NOT EXISTS idx_query_projection_dirty_type + ON query_projection_dirty (tenant, entity_type, entity_id); + +-- Bridge writes from older binaries during the required drain/upgrade cutover. +-- Source triggers cover old journal/snapshot writers. The catalog trigger +-- covers their delayed asynchronous projector after a new reader has repaired +-- and cleared an earlier marker. Older readers still cannot interpret the new +-- materialization control event, so ADR-0192 requires draining them before the +-- new binary is enabled; these triggers do not replace that reader barrier. +CREATE OR REPLACE FUNCTION temper_mark_query_projection_dirty() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +BEGIN + IF TG_OP <> 'INSERT' THEN + INSERT INTO query_projection_dirty (tenant, entity_type, entity_id) + VALUES (OLD.tenant, OLD.entity_type, OLD.entity_id) + ON CONFLICT (tenant, entity_type, entity_id) DO NOTHING; + END IF; + IF TG_OP <> 'DELETE' THEN + INSERT INTO query_projection_dirty (tenant, entity_type, entity_id) + VALUES (NEW.tenant, NEW.entity_type, NEW.entity_id) + ON CONFLICT (tenant, entity_type, entity_id) DO NOTHING; + END IF; + RETURN NULL; +END; +$$; + +DROP TRIGGER IF EXISTS events_mark_query_projection_dirty ON events; +CREATE TRIGGER events_mark_query_projection_dirty +AFTER INSERT OR UPDATE OR DELETE ON events +FOR EACH ROW EXECUTE FUNCTION temper_mark_query_projection_dirty(); + +DROP TRIGGER IF EXISTS snapshots_mark_query_projection_dirty ON snapshots; +CREATE TRIGGER snapshots_mark_query_projection_dirty +AFTER INSERT OR UPDATE OR DELETE ON snapshots +FOR EACH ROW EXECUTE FUNCTION temper_mark_query_projection_dirty(); + +DROP TRIGGER IF EXISTS catalog_mark_query_projection_dirty ON entity_catalog; +CREATE TRIGGER catalog_mark_query_projection_dirty +AFTER INSERT OR UPDATE OR DELETE ON entity_catalog +FOR EACH ROW EXECUTE FUNCTION temper_mark_query_projection_dirty(); + +-- This migration is the publication boundary for the ledger. Seed every +-- authoritative source row so missing projections are repaired, and every +-- existing catalog row so stale projections are fenced before native reads. +-- Catalog-only compatibility rows are source-fenced acknowledgements: repair +-- clears their marker without deleting the row. +INSERT INTO query_projection_dirty (tenant, entity_type, entity_id) +SELECT tenant, entity_type, entity_id FROM events +UNION +SELECT tenant, entity_type, entity_id FROM snapshots +UNION +SELECT tenant, entity_type, entity_id FROM entity_catalog +ON CONFLICT (tenant, entity_type, entity_id) DO NOTHING; diff --git a/crates/temper-store-postgres/migrations/0015_key_index_contract_activation.sql b/crates/temper-store-postgres/migrations/0015_key_index_contract_activation.sql new file mode 100644 index 000000000..dc55a19e4 --- /dev/null +++ b/crates/temper-store-postgres/migrations/0015_key_index_contract_activation.sql @@ -0,0 +1,23 @@ +-- ADR-0192 / ARN-238: reject delayed writers from an older hot-swapped spec. +-- +-- `key_set` tracks the latest contract used for reconciliation and revision +-- fencing. `activated_key_set` is stronger: once a spec activation publishes +-- it, a live writer carrying any other signature must fail instead of changing +-- the contract back and reintroducing rows that the new spec released. +ALTER TABLE key_index_contract_state + ADD COLUMN IF NOT EXISTS activated_key_set TEXT, + ADD COLUMN IF NOT EXISTS activated_spec_fingerprint TEXT, + ADD COLUMN IF NOT EXISTS activation_epoch BIGINT NOT NULL DEFAULT 0 + CHECK (activation_epoch >= 0); + +-- Composite retries must not rescan an ever-growing parent journal. The claim +-- and content hash commit in the same transaction as every participating +-- stream, so a retry is a constant-time exact no-op and key reuse cannot hide +-- different work. +CREATE TABLE IF NOT EXISTS persistence_batch_idempotency ( + persistence_id TEXT NOT NULL, + idempotency_key TEXT NOT NULL, + intent_hash TEXT NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (persistence_id, idempotency_key) +); diff --git a/crates/temper-store-postgres/src/data_only_create.rs b/crates/temper-store-postgres/src/data_only_create.rs index c1b3e2c6b..5262ec4a3 100644 --- a/crates/temper-store-postgres/src/data_only_create.rs +++ b/crates/temper-store-postgres/src/data_only_create.rs @@ -3,7 +3,7 @@ use std::time::Instant; use sqlx::Acquire; -use temper_runtime::persistence::{PersistenceEnvelope, PersistenceError}; +use temper_runtime::persistence::{EntityKeyRow, PersistenceEnvelope, PersistenceError}; use crate::PostgresEventStore; use crate::metrics::{ @@ -12,6 +12,11 @@ use crate::metrics::{ record_postgres_transaction_begin_duration, record_postgres_transaction_commit_duration, }; use crate::platform::{canonical_projection_status, json_hash, scalar_index_fields, storage_error}; +use crate::store::{ + KeyContractUse, clear_query_projection_dirty, event_stream_lock_key, + invalidate_key_coverage_for_unreconciled_append, lock_event_stream, lock_key_contract, + reconcile_key_contract_state, +}; const DATA_ONLY_CREATE_OPERATION: &str = "data_only_create"; @@ -65,11 +70,51 @@ impl PostgresEventStore { fields: &serde_json::Value, state: &serde_json::Value, event: &PersistenceEnvelope, + ) -> Result { + self.create_data_only_entity_native_with_state_and_keys( + tenant, + entity_type, + entity_id, + status, + fields, + state, + event, + &[], + false, + None, + ) + .await + } + + /// Atomically insert the first data-only event, query projection, and exact + /// declared-key ownership set. + #[expect( + clippy::too_many_arguments, + reason = "native data-only create storage boundary" + )] + pub async fn create_data_only_entity_native_with_state_and_keys( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + status: &str, + fields: &serde_json::Value, + state: &serde_json::Value, + event: &PersistenceEnvelope, + key_rows: &[EntityKeyRow], + reconcile_keys: bool, + key_set_signature: Option<&str>, ) -> Result { assert_eq!( event.sequence_nr, 1, "native data-only create requires first event sequence" ); + if !reconcile_keys && !key_rows.is_empty() { + return Err(PersistenceError::Storage( + "native data-only create supplied key rows without exact reconciliation" + .to_string(), + )); + } let status = canonical_projection_status(status, state); let projection_hash = json_hash(fields); let (new_index, indexed_fields, skipped_fields) = scalar_index_fields(fields); @@ -120,6 +165,63 @@ impl PostgresEventStore { } }; + lock_key_contract(&mut tx, tenant, entity_type).await?; + let lock_key = event_stream_lock_key(tenant, entity_type, entity_id); + lock_event_stream(&mut tx, &lock_key).await?; + + let snapshot: Option<(i64,)> = crate::dbm::postgres_query_as!( + "SELECT sequence_nr FROM snapshots \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ + FOR UPDATE", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_optional(&mut *tx) + .await + .map_err(storage_error)?; + if snapshot.is_some() { + return Err(PersistenceError::SnapshotGenerationChanged); + } + + if reconcile_keys { + for key in key_rows { + let holder: Option<(String,)> = crate::dbm::postgres_query_as!( + "SELECT entity_id FROM entity_key_index \ + WHERE tenant = $1 AND entity_type = $2 AND key_name = $3 AND key_hash = $4", + ) + .bind(tenant) + .bind(entity_type) + .bind(&key.key_name) + .bind(&key.key_hash) + .fetch_optional(&mut *tx) + .await + .map_err(storage_error)?; + if let Some((existing,)) = holder + && existing != entity_id + { + return Err(PersistenceError::Storage(format!( + "duplicate declared key '{}' for {entity_type}: held by {existing}", + key.key_name + ))); + } + } + } + + if reconcile_keys { + reconcile_key_contract_state( + &mut tx, + tenant, + entity_type, + key_set_signature, + None, + KeyContractUse::LiveWrite, + ) + .await?; + } else { + invalidate_key_coverage_for_unreconciled_append(&mut tx, tenant, entity_type).await?; + } + let metadata_json = serde_json::to_value(&event.metadata) .map_err(|e| PersistenceError::Serialization(e.to_string()))?; if let Err(e) = crate::dbm::postgres_query!( @@ -191,6 +293,40 @@ impl PostgresEventStore { .map_err(storage_error)?; } + if reconcile_keys { + crate::dbm::postgres_query!( + "DELETE FROM entity_key_index \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .execute(&mut *tx) + .await + .map_err(storage_error)?; + for key in key_rows { + crate::dbm::postgres_query!( + "INSERT INTO entity_key_index \ + (tenant, entity_type, key_name, key_hash, entity_id, sequence_nr) \ + VALUES ($1, $2, $3, $4, $5, 1)", + ) + .bind(tenant) + .bind(entity_type) + .bind(&key.key_name) + .bind(&key.key_hash) + .bind(entity_id) + .execute(&mut *tx) + .await + .map_err(storage_error)?; + } + } + + // The event and exact catalog/EAV projection were installed under the + // same stream transaction. Migration triggers conservatively mark both + // writes dirty for mixed-version safety; close that marker only after + // every projection row above is complete. + clear_query_projection_dirty(&mut tx, tenant, entity_type, entity_id).await?; + let commit_started = Instant::now(); tx.commit().await.map_err(|e| { record_postgres_transaction_commit_duration( diff --git a/crates/temper-store-postgres/src/lib.rs b/crates/temper-store-postgres/src/lib.rs index af25937ce..e4024c43b 100644 --- a/crates/temper-store-postgres/src/lib.rs +++ b/crates/temper-store-postgres/src/lib.rs @@ -29,9 +29,9 @@ pub use platform::{ PostgresActionStats, PostgresAgentSummary, PostgresDesignTimeEventRow, PostgresEvolutionRecordRow, PostgresFeatureRequestRow, PostgresInstalledAppRow, PostgresOtsTrajectoryParams, PostgresOtsTrajectoryRow, PostgresPolicyDenialPatternRow, - PostgresPolicyRow, PostgresProjectedEntityFieldsRow, PostgresPublishedArtifactRow, - PostgresPublishedArtifactUpsert, PostgresQueuedOtsTrajectoryRow, PostgresSecretRow, - PostgresSpecRow, PostgresSpecVerificationUpdate, PostgresTrajectoryInsert, + PostgresPolicyGenerationWrite, PostgresPolicyRow, PostgresProjectedEntityFieldsRow, + PostgresPublishedArtifactRow, PostgresPublishedArtifactUpsert, PostgresQueuedOtsTrajectoryRow, + PostgresSecretRow, PostgresSpecRow, PostgresSpecVerificationUpdate, PostgresTrajectoryInsert, PostgresTrajectoryRow, PostgresTrajectoryStats, PostgresUnmetIntentAggRow, PostgresWasmInvocationInsert, PostgresWasmInvocationRow, PostgresWasmModuleMetadataRow, PostgresWasmModuleRow, diff --git a/crates/temper-store-postgres/src/migration.rs b/crates/temper-store-postgres/src/migration.rs index 5a40c7493..ba7b18cc2 100644 --- a/crates/temper-store-postgres/src/migration.rs +++ b/crates/temper-store-postgres/src/migration.rs @@ -103,6 +103,41 @@ mod tests { ); } + #[test] + fn projection_dirty_migration_fences_pre_existing_sources_and_catalog_rows() { + let migration = + include_str!("../migrations/0014_query_projection_dirty.sql").to_ascii_lowercase(); + for source in ["from events", "from snapshots", "from entity_catalog"] { + assert!( + migration.contains(source), + "projection dirty migration must seed {source}" + ); + } + assert!( + migration.contains("on conflict (tenant, entity_type, entity_id) do nothing"), + "projection dirty migration seed must be idempotent" + ); + for trigger in [ + "events_mark_query_projection_dirty", + "snapshots_mark_query_projection_dirty", + "catalog_mark_query_projection_dirty", + ] { + assert!( + migration.contains(trigger), + "projection dirty migration must install the {trigger} mixed-version fence" + ); + } + } + + #[test] + fn key_contract_activation_migration_adds_monotonic_epoch_fence() { + let migration = include_str!("../migrations/0015_key_index_contract_activation.sql") + .to_ascii_lowercase(); + assert!(migration.contains("activated_key_set text")); + assert!(migration.contains("activation_epoch bigint not null default 0")); + assert!(migration.contains("check (activation_epoch >= 0)")); + } + #[test] fn migration_sql_is_idempotent() { // Both schemas must use IF NOT EXISTS so repeated execution is safe. diff --git a/crates/temper-store-postgres/src/platform.rs b/crates/temper-store-postgres/src/platform.rs index 322e61fe9..dab4afc1b 100644 --- a/crates/temper-store-postgres/src/platform.rs +++ b/crates/temper-store-postgres/src/platform.rs @@ -5,7 +5,7 @@ use std::time::{Duration, Instant}; use sha2::{Digest, Sha256}; use sqlx::{Acquire, Postgres, Row, Transaction}; -use temper_runtime::persistence::PersistenceError; +use temper_runtime::persistence::{PersistenceError, ProjectionSourceFence}; use crate::PostgresEventStore; use crate::metrics::{ @@ -13,8 +13,17 @@ use crate::metrics::{ record_postgres_projection_index_fields, record_postgres_projection_index_reconciliation, record_postgres_transaction_begin_duration, record_postgres_transaction_commit_duration, }; +use crate::store::{ + DerivedWriteSource, clear_query_projection_dirty, event_stream_lock_key, + invalidate_key_coverage_for_derived_write, lock_event_stream, lock_key_contract, + mark_query_projection_dirty, +}; +mod query_projection_read; +mod query_projection_repair; +mod query_projection_write; mod rows; +mod spec_publication; use rows::*; const DISTINCT_RESOURCE_IDS_BUDGET: usize = 100; @@ -32,6 +41,36 @@ const QUERY_PROJECTION_REMOVE_OPERATION: &str = "query_projection_remove"; pub(crate) type ScalarFieldIndex = BTreeMap; type CatalogProjectionFingerprint = (String, String, i64); +async fn projection_snapshot_source_matches( + tx: &mut Transaction<'_, Postgres>, + tenant: &str, + entity_type: &str, + entity_id: &str, + source: ProjectionSourceFence<'_>, +) -> Result { + let current_snapshot: Option<(i64, Vec)> = crate::dbm::postgres_query_as!( + "SELECT sequence_nr, state FROM snapshots \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ + FOR UPDATE", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_optional(&mut **tx) + .await + .map_err(storage_error)?; + Ok( + match (source.expected_snapshot, current_snapshot.as_ref()) { + (None, None) => true, + (Some(expected), Some((sequence_nr, state))) => { + u64::try_from(*sequence_nr).ok() == Some(expected.sequence_nr) + && state.as_slice() == expected.state + } + _ => false, + }, + ) +} + pub(crate) fn canonical_projection_status<'a>( fallback: &'a str, state: &'a serde_json::Value, @@ -255,6 +294,19 @@ pub struct PostgresPolicyRow { pub enabled: bool, } +/// One entry in an atomically published tenant policy generation. +#[derive(Debug, Clone)] +pub struct PostgresPolicyGenerationWrite<'a> { + /// Stable row identifier within the tenant. + pub policy_id: &'a str, + /// Raw Cedar source for the row. + pub cedar_text: &'a str, + /// Whether the row participates in the live generation. + pub enabled: bool, + /// Identity responsible for the row mutation. + pub created_by: &'a str, +} + #[derive(Debug, Clone, serde::Serialize)] pub struct PostgresPolicyDenialPatternRow { pub tenant: String, @@ -520,527 +572,6 @@ impl PostgresEventStore { Ok(()) } - pub async fn upsert_query_projection( - &self, - tenant: &str, - entity_type: &str, - entity_id: &str, - status: &str, - fields: &serde_json::Value, - sequence_nr: u64, - ) -> Result<(), PersistenceError> { - let state = serde_json::json!({ - "entity_type": entity_type, - "entity_id": entity_id, - "status": status, - "item_count": 0, - "counters": {}, - "booleans": {}, - "lists": {}, - "fields": fields, - "events": [], - "total_event_count": sequence_nr, - "sequence_nr": sequence_nr - }); - self.upsert_query_projection_with_state( - tenant, - entity_type, - entity_id, - status, - fields, - &state, - sequence_nr, - ) - .await - } - - #[expect(clippy::too_many_arguments, reason = "projection upsert boundary")] - pub async fn upsert_query_projection_with_state( - &self, - tenant: &str, - entity_type: &str, - entity_id: &str, - status: &str, - fields: &serde_json::Value, - state: &serde_json::Value, - sequence_nr: u64, - ) -> Result<(), PersistenceError> { - let status = canonical_projection_status(status, state); - let projection_hash = json_hash(fields); - let (new_index, indexed_fields, skipped_fields) = scalar_index_fields(fields); - let mut transaction_timer = - PostgresTransactionTimer::start(QUERY_PROJECTION_UPSERT_OPERATION); - let acquire_started = Instant::now(); - let mut conn = match self.pool().acquire().await { - Ok(conn) => { - record_postgres_pool_acquire_duration( - acquire_started.elapsed(), - QUERY_PROJECTION_UPSERT_OPERATION, - "ok", - ); - conn - } - Err(e) => { - record_postgres_pool_acquire_duration( - acquire_started.elapsed(), - QUERY_PROJECTION_UPSERT_OPERATION, - "error", - ); - return Err(storage_error(e)); - } - }; - let begin_started = Instant::now(); - let mut tx = match conn.begin().await { - Ok(tx) => { - record_postgres_transaction_begin_duration( - begin_started.elapsed(), - QUERY_PROJECTION_UPSERT_OPERATION, - "ok", - ); - tx - } - Err(e) => { - record_postgres_transaction_begin_duration( - begin_started.elapsed(), - QUERY_PROJECTION_UPSERT_OPERATION, - "error", - ); - return Err(storage_error(e)); - } - }; - - let previous_catalog: Option = - crate::dbm::postgres_query_as!( - "SELECT status, projection_hash, sequence_nr \ - FROM entity_catalog \ - WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ - FOR UPDATE", - ) - .bind(tenant) - .bind(entity_type) - .bind(entity_id) - .fetch_optional(&mut *tx) - .await - .map_err(storage_error)?; - - let new_sequence_nr = sequence_nr as i64; - let previous_catalog = if previous_catalog - .as_ref() - .is_some_and(|(_, _, existing_sequence)| *existing_sequence > new_sequence_nr) - { - let commit_started = Instant::now(); - tx.commit().await.map_err(|e| { - record_postgres_transaction_commit_duration( - commit_started.elapsed(), - QUERY_PROJECTION_UPSERT_OPERATION, - "error", - ); - storage_error(e) - })?; - record_postgres_transaction_commit_duration( - commit_started.elapsed(), - QUERY_PROJECTION_UPSERT_OPERATION, - "ok", - ); - record_postgres_projection_index_fields( - QUERY_PROJECTION_UPSERT_OPERATION, - entity_type, - indexed_fields, - skipped_fields, - ); - record_postgres_projection_index_reconciliation( - QUERY_PROJECTION_UPSERT_OPERATION, - "stale_skipped", - ); - transaction_timer.set_outcome("stale_skipped"); - return Ok(()); - } else if previous_catalog.is_some() { - update_query_projection_catalog_row( - &mut tx, - QueryProjectionCatalogUpdate { - tenant, - entity_type, - entity_id, - status, - fields, - state, - sequence_nr, - projection_hash: projection_hash.as_str(), - }, - ) - .await?; - previous_catalog - } else { - let inserted: Option = crate::dbm::postgres_query_scalar!( - "INSERT INTO entity_catalog \ - (tenant, entity_type, entity_id, status, fields, state, sequence_nr, projection_version, projection_hash, updated_at) \ - VALUES ($1, $2, $3, $4, $5, $6, $7, 2, $8, now()) \ - ON CONFLICT (tenant, entity_type, entity_id) DO NOTHING \ - RETURNING 1", - ) - .bind(tenant) - .bind(entity_type) - .bind(entity_id) - .bind(status) - .bind(fields) - .bind(state) - .bind(sequence_nr as i64) - .bind(projection_hash.as_str()) - .fetch_optional(&mut *tx) - .await - .map_err(storage_error)?; - - if inserted.is_some() { - None - } else { - let raced_catalog: Option = - crate::dbm::postgres_query_as!( - "SELECT status, projection_hash, sequence_nr \ - FROM entity_catalog \ - WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ - FOR UPDATE", - ) - .bind(tenant) - .bind(entity_type) - .bind(entity_id) - .fetch_optional(&mut *tx) - .await - .map_err(storage_error)?; - if raced_catalog - .as_ref() - .is_some_and(|(_, _, existing_sequence)| *existing_sequence > new_sequence_nr) - { - let commit_started = Instant::now(); - tx.commit().await.map_err(|e| { - record_postgres_transaction_commit_duration( - commit_started.elapsed(), - QUERY_PROJECTION_UPSERT_OPERATION, - "error", - ); - storage_error(e) - })?; - record_postgres_transaction_commit_duration( - commit_started.elapsed(), - QUERY_PROJECTION_UPSERT_OPERATION, - "ok", - ); - record_postgres_projection_index_fields( - QUERY_PROJECTION_UPSERT_OPERATION, - entity_type, - indexed_fields, - skipped_fields, - ); - record_postgres_projection_index_reconciliation( - QUERY_PROJECTION_UPSERT_OPERATION, - "stale_skipped", - ); - transaction_timer.set_outcome("stale_skipped"); - return Ok(()); - } - update_query_projection_catalog_row( - &mut tx, - QueryProjectionCatalogUpdate { - tenant, - entity_type, - entity_id, - status, - fields, - state, - sequence_nr, - projection_hash: projection_hash.as_str(), - }, - ) - .await?; - raced_catalog - } - }; - - let should_reconcile_index = - previous_catalog - .as_ref() - .is_none_or(|(old_status, old_hash, _)| { - old_status.as_str() != status || old_hash.as_str() != projection_hash.as_str() - }); - let reconciliation_path = if should_reconcile_index { - if previous_catalog.is_some() { - "diff" - } else { - "insert" - } - } else { - "skipped_unchanged" - }; - - if should_reconcile_index { - reconcile_query_projection_field_index( - &mut tx, - tenant, - entity_type, - entity_id, - status, - &new_index, - ) - .await?; - } - - let commit_started = Instant::now(); - tx.commit().await.map_err(|e| { - record_postgres_transaction_commit_duration( - commit_started.elapsed(), - QUERY_PROJECTION_UPSERT_OPERATION, - "error", - ); - storage_error(e) - })?; - record_postgres_transaction_commit_duration( - commit_started.elapsed(), - QUERY_PROJECTION_UPSERT_OPERATION, - "ok", - ); - record_postgres_projection_index_fields( - QUERY_PROJECTION_UPSERT_OPERATION, - entity_type, - indexed_fields, - skipped_fields, - ); - record_postgres_projection_index_reconciliation( - QUERY_PROJECTION_UPSERT_OPERATION, - reconciliation_path, - ); - transaction_timer.set_outcome("ok"); - Ok(()) - } - - pub async fn remove_query_projection( - &self, - tenant: &str, - entity_type: &str, - entity_id: &str, - ) -> Result<(), PersistenceError> { - let mut transaction_timer = - PostgresTransactionTimer::start(QUERY_PROJECTION_REMOVE_OPERATION); - let acquire_started = Instant::now(); - let mut conn = match self.pool().acquire().await { - Ok(conn) => { - record_postgres_pool_acquire_duration( - acquire_started.elapsed(), - QUERY_PROJECTION_REMOVE_OPERATION, - "ok", - ); - conn - } - Err(e) => { - record_postgres_pool_acquire_duration( - acquire_started.elapsed(), - QUERY_PROJECTION_REMOVE_OPERATION, - "error", - ); - return Err(storage_error(e)); - } - }; - let begin_started = Instant::now(); - let mut tx = match conn.begin().await { - Ok(tx) => { - record_postgres_transaction_begin_duration( - begin_started.elapsed(), - QUERY_PROJECTION_REMOVE_OPERATION, - "ok", - ); - tx - } - Err(e) => { - record_postgres_transaction_begin_duration( - begin_started.elapsed(), - QUERY_PROJECTION_REMOVE_OPERATION, - "error", - ); - return Err(storage_error(e)); - } - }; - crate::dbm::postgres_query!( - "DELETE FROM entity_catalog WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", - ) - .bind(tenant) - .bind(entity_type) - .bind(entity_id) - .execute(&mut *tx) - .await - .map_err(storage_error)?; - crate::dbm::postgres_query!("DELETE FROM entity_field_index WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3") - .bind(tenant) - .bind(entity_type) - .bind(entity_id) - .execute(&mut *tx) - .await - .map_err(storage_error)?; - let commit_started = Instant::now(); - tx.commit().await.map_err(|e| { - record_postgres_transaction_commit_duration( - commit_started.elapsed(), - QUERY_PROJECTION_REMOVE_OPERATION, - "error", - ); - storage_error(e) - })?; - record_postgres_transaction_commit_duration( - commit_started.elapsed(), - QUERY_PROJECTION_REMOVE_OPERATION, - "ok", - ); - transaction_timer.set_outcome("ok"); - Ok(()) - } - - pub async fn query_field_index( - &self, - tenant: &str, - entity_type: &str, - where_clause: &str, - params: Vec, - ) -> Result, PersistenceError> { - let clause = postgres_placeholders(where_clause, params.len() + 2); - let sql = format!( - "SELECT entity_id FROM entity_catalog \ - WHERE tenant = $1 AND entity_type = $2 AND ({clause}) \ - ORDER BY entity_id" - ); - let tagged_sql = crate::dbm::tag_sql(&sql); - let mut query = sqlx::query_scalar::<_, String>(tagged_sql.as_ref()) - .bind(tenant) - .bind(entity_type); - for param in params { - query = query.bind(param); - } - query.fetch_all(self.pool()).await.map_err(storage_error) - } - - pub async fn load_query_projection_fields_many( - &self, - tenant: &str, - entity_type: &str, - entity_ids: &[String], - field_names: &[&str], - ) -> Result, PersistenceError> { - if entity_ids.is_empty() { - return Ok(Vec::new()); - } - - let requested_fields: BTreeSet = field_names - .iter() - .map(|field| (*field).to_string()) - .collect(); - let field_names = requested_fields.iter().cloned().collect::>(); - let rows = crate::dbm::postgres_query!( - "SELECT c.entity_id, c.status, f.field_name, f.field_value \ - FROM entity_catalog c \ - LEFT JOIN entity_field_index f \ - ON c.tenant = f.tenant \ - AND c.entity_type = f.entity_type \ - AND c.entity_id = f.entity_id \ - AND f.field_name = ANY($4) \ - WHERE c.tenant = $1 \ - AND c.entity_type = $2 \ - AND c.entity_id = ANY($3) \ - ORDER BY c.entity_id, f.field_name", - ) - .bind(tenant) - .bind(entity_type) - .bind(entity_ids) - .bind(&field_names) - .fetch_all(self.pool()) - .await - .map_err(storage_error)?; - - let mut by_entity = BTreeMap::::new(); - for row in rows { - let entity_id: String = row.get("entity_id"); - let status: String = row.get("status"); - let field_name: Option = row.get("field_name"); - let field_value: Option = row.get("field_value"); - let entry = by_entity.entry(entity_id.clone()).or_insert_with(|| { - PostgresProjectedEntityFieldsRow { - entity_id: entity_id.clone(), - status, - fields: requested_fields - .iter() - .map(|field| (field.clone(), None)) - .collect(), - } - }); - if let Some(field_name) = field_name - && requested_fields.contains(&field_name) - { - entry.fields.insert(field_name, field_value); - } - } - - Ok(entity_ids - .iter() - .filter_map(|entity_id| by_entity.remove(entity_id)) - .collect()) - } - - pub async fn projected_entity_counts_by_tenant( - &self, - ) -> Result, PersistenceError> { - let rows: Vec<(String, i64)> = crate::dbm::postgres_query_as!( - "SELECT tenant, COUNT(*)::bigint FROM entity_catalog GROUP BY tenant ORDER BY tenant", - ) - .fetch_all(self.pool()) - .await - .map_err(storage_error)?; - Ok(rows - .into_iter() - .map(|(tenant, count)| (tenant, count as u64)) - .collect()) - } - - /// Batch-load full entity catalog rows for a list of entity IDs. - /// - /// Returns only rows that exist in the catalog. IDs without a row in the - /// projection are silently omitted from the result, leaving the caller - /// free to fall back to the actor path on a per-id basis. - pub async fn load_entity_catalog_rows_pg( - &self, - tenant: &str, - entity_type: &str, - entity_ids: &[String], - ) -> Result, PersistenceError> { - if entity_ids.is_empty() { - return Ok(Vec::new()); - } - let rows: Vec<( - String, - String, - serde_json::Value, - Option, - i64, - )> = crate::dbm::postgres_query_as!( - "SELECT entity_id, status, fields, state, sequence_nr \ - FROM entity_catalog \ - WHERE tenant = $1 AND entity_type = $2 AND entity_id = ANY($3) \ - ORDER BY entity_id", - ) - .bind(tenant) - .bind(entity_type) - .bind(entity_ids) - .fetch_all(self.pool()) - .await - .map_err(storage_error)?; - Ok(rows - .into_iter() - .map(|(entity_id, status, fields, state, seq)| { - crate::platform::PostgresEntityCatalogRow { - entity_id, - status, - fields, - state, - sequence_nr: seq.max(0) as u64, - } - }) - .collect()) - } - pub async fn upsert_spec( &self, tenant: &str, @@ -1190,6 +721,73 @@ impl PostgresEventStore { .map_err(storage_error) } + /// Load one tenant's compatibility policy projection. + pub async fn load_tenant_policy( + &self, + tenant: &str, + ) -> Result, PersistenceError> { + crate::dbm::postgres_query_scalar!( + "SELECT policy_text FROM tenant_policies WHERE tenant = $1" + ) + .bind(tenant) + .fetch_optional(self.pool()) + .await + .map_err(storage_error) + } + + /// Atomically replace both representations of one tenant policy generation. + pub async fn replace_policy_generation( + &self, + tenant: &str, + entries: &[PostgresPolicyGenerationWrite<'_>], + compatibility_text: &str, + ) -> Result<(), PersistenceError> { + let mut policy_ids = BTreeSet::new(); + for entry in entries { + if entry.policy_id.is_empty() || !policy_ids.insert(entry.policy_id) { + return Err(PersistenceError::Storage(format!( + "policy generation for tenant '{tenant}' contains an empty or duplicate policy id" + ))); + } + } + + let mut tx = self.pool().begin().await.map_err(storage_error)?; + crate::dbm::postgres_query!("DELETE FROM policies WHERE tenant = $1") + .bind(tenant) + .execute(&mut *tx) + .await + .map_err(storage_error)?; + for entry in entries { + let policy_hash = compute_policy_hash(entry.cedar_text); + crate::dbm::postgres_query!( + "INSERT INTO policies \ + (tenant, policy_id, cedar_text, policy_hash, created_at, created_by, enabled) \ + VALUES ($1, $2, $3, $4, now(), $5, $6)" + ) + .bind(tenant) + .bind(entry.policy_id) + .bind(entry.cedar_text) + .bind(policy_hash) + .bind(entry.created_by) + .bind(entry.enabled) + .execute(&mut *tx) + .await + .map_err(storage_error)?; + } + crate::dbm::postgres_query!( + "INSERT INTO tenant_policies (tenant, policy_text, updated_at) \ + VALUES ($1, $2, now()) \ + ON CONFLICT (tenant) DO UPDATE SET \ + policy_text = EXCLUDED.policy_text, updated_at = now()" + ) + .bind(tenant) + .bind(compatibility_text) + .execute(&mut *tx) + .await + .map_err(storage_error)?; + tx.commit().await.map_err(storage_error) + } + pub async fn save_policy( &self, tenant: &str, @@ -1198,8 +796,8 @@ impl PostgresEventStore { created_by: &str, ) -> Result { let policy_hash = compute_policy_hash(cedar_text); - let existing_hash: Option = crate::dbm::postgres_query_scalar!( - "SELECT policy_hash FROM policies WHERE tenant = $1 AND policy_id = $2", + let existing: Option<(String, bool)> = crate::dbm::postgres_query_as!( + "SELECT policy_hash, enabled FROM policies WHERE tenant = $1 AND policy_id = $2", ) .bind(tenant) .bind(policy_id) @@ -1207,7 +805,10 @@ impl PostgresEventStore { .await .map_err(storage_error)?; - if existing_hash.as_deref() == Some(policy_hash.as_str()) { + if existing + .as_ref() + .is_some_and(|(hash, enabled)| hash == &policy_hash && *enabled) + { return Ok(false); } @@ -1219,7 +820,8 @@ impl PostgresEventStore { cedar_text = EXCLUDED.cedar_text, \ policy_hash = EXCLUDED.policy_hash, \ created_by = EXCLUDED.created_by, \ - created_at = now()", + created_at = now(), \ + enabled = true", ) .bind(tenant) .bind(policy_id) @@ -1305,6 +907,34 @@ impl PostgresEventStore { Ok(result.rows_affected() > 0) } + /// Atomically replace the text and enabled state of one Cedar policy. + pub async fn replace_policy( + &self, + tenant: &str, + policy_id: &str, + cedar_text: &str, + enabled: bool, + created_by: &str, + ) -> Result { + let policy_hash = compute_policy_hash(cedar_text); + let result = crate::dbm::postgres_query!( + "UPDATE policies \ + SET cedar_text = $3, policy_hash = $4, enabled = $5, \ + created_by = $6, created_at = now() \ + WHERE tenant = $1 AND policy_id = $2", + ) + .bind(tenant) + .bind(policy_id) + .bind(cedar_text) + .bind(&policy_hash) + .bind(enabled) + .bind(created_by) + .execute(self.pool()) + .await + .map_err(storage_error)?; + Ok(result.rows_affected() > 0) + } + pub async fn delete_policy( &self, tenant: &str, diff --git a/crates/temper-store-postgres/src/platform/query_projection_read.rs b/crates/temper-store-postgres/src/platform/query_projection_read.rs new file mode 100644 index 000000000..b3e3e7827 --- /dev/null +++ b/crates/temper-store-postgres/src/platform/query_projection_read.rs @@ -0,0 +1,176 @@ +//! PostgreSQL query-projection reads. + +use super::*; + +impl PostgresEventStore { + pub async fn query_field_index( + &self, + tenant: &str, + entity_type: &str, + where_clause: &str, + params: Vec, + ) -> Result, PersistenceError> { + let clause = postgres_placeholders(where_clause, params.len() + 2); + let sql = format!( + "SELECT entity_id FROM entity_catalog \ + WHERE tenant = $1 AND entity_type = $2 AND ({clause}) \ + ORDER BY entity_id" + ); + let tagged_sql = crate::dbm::tag_sql(&sql); + let mut query = sqlx::query_scalar::<_, String>(tagged_sql.as_ref()) + .bind(tenant) + .bind(entity_type); + for param in params { + query = query.bind(param); + } + query.fetch_all(self.pool()).await.map_err(storage_error) + } + + pub async fn load_query_projection_fields_many( + &self, + tenant: &str, + entity_type: &str, + entity_ids: &[String], + field_names: &[&str], + ) -> Result, PersistenceError> { + if entity_ids.is_empty() { + return Ok(Vec::new()); + } + + let requested_fields: BTreeSet = field_names + .iter() + .map(|field| (*field).to_string()) + .collect(); + let field_names = requested_fields.iter().cloned().collect::>(); + let rows = crate::dbm::postgres_query!( + "SELECT c.entity_id, c.status, f.field_name, f.field_value \ + FROM entity_catalog c \ + LEFT JOIN entity_field_index f \ + ON c.tenant = f.tenant \ + AND c.entity_type = f.entity_type \ + AND c.entity_id = f.entity_id \ + AND f.field_name = ANY($4) \ + WHERE c.tenant = $1 \ + AND c.entity_type = $2 \ + AND c.entity_id = ANY($3) \ + ORDER BY c.entity_id, f.field_name", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_ids) + .bind(&field_names) + .fetch_all(self.pool()) + .await + .map_err(storage_error)?; + + let mut by_entity = BTreeMap::::new(); + for row in rows { + let entity_id: String = row.get("entity_id"); + let status: String = row.get("status"); + let field_name: Option = row.get("field_name"); + let field_value: Option = row.get("field_value"); + let entry = by_entity.entry(entity_id.clone()).or_insert_with(|| { + PostgresProjectedEntityFieldsRow { + entity_id: entity_id.clone(), + status, + fields: requested_fields + .iter() + .map(|field| (field.clone(), None)) + .collect(), + } + }); + if let Some(field_name) = field_name + && requested_fields.contains(&field_name) + { + entry.fields.insert(field_name, field_value); + } + } + + Ok(entity_ids + .iter() + .filter_map(|entity_id| by_entity.remove(entity_id)) + .collect()) + } + + pub async fn projected_entity_counts_by_tenant( + &self, + ) -> Result, PersistenceError> { + let rows: Vec<(String, i64)> = crate::dbm::postgres_query_as!( + "SELECT tenant, COUNT(*)::bigint FROM entity_catalog GROUP BY tenant ORDER BY tenant", + ) + .fetch_all(self.pool()) + .await + .map_err(storage_error)?; + Ok(rows + .into_iter() + .map(|(tenant, count)| (tenant, count as u64)) + .collect()) + } + + /// Return a bounded set of entities requiring exact-source projection repair. + pub async fn dirty_query_projection_entity_ids( + &self, + tenant: &str, + entity_type: &str, + limit: usize, + ) -> Result, PersistenceError> { + let limit = i64::try_from(limit).unwrap_or(i64::MAX); + crate::dbm::postgres_query_scalar!( + "SELECT entity_id FROM query_projection_dirty \ + WHERE tenant = $1 AND entity_type = $2 \ + ORDER BY entity_id LIMIT $3", + ) + .bind(tenant) + .bind(entity_type) + .bind(limit) + .fetch_all(self.pool()) + .await + .map_err(storage_error) + } + + /// Batch-load full entity catalog rows for a list of entity IDs. + /// + /// Returns only rows that exist in the catalog. IDs without a row in the + /// projection are silently omitted from the result, leaving the caller + /// free to fall back to the actor path on a per-id basis. + pub async fn load_entity_catalog_rows_pg( + &self, + tenant: &str, + entity_type: &str, + entity_ids: &[String], + ) -> Result, PersistenceError> { + if entity_ids.is_empty() { + return Ok(Vec::new()); + } + let rows: Vec<( + String, + String, + serde_json::Value, + Option, + i64, + )> = crate::dbm::postgres_query_as!( + "SELECT entity_id, status, fields, state, sequence_nr \ + FROM entity_catalog \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = ANY($3) \ + ORDER BY entity_id", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_ids) + .fetch_all(self.pool()) + .await + .map_err(storage_error)?; + Ok(rows + .into_iter() + .map(|(entity_id, status, fields, state, seq)| { + crate::platform::PostgresEntityCatalogRow { + entity_id, + status, + fields, + state, + sequence_nr: seq.max(0) as u64, + } + }) + .collect()) + } +} diff --git a/crates/temper-store-postgres/src/platform/query_projection_repair.rs b/crates/temper-store-postgres/src/platform/query_projection_repair.rs new file mode 100644 index 000000000..6066d80ed --- /dev/null +++ b/crates/temper-store-postgres/src/platform/query_projection_repair.rs @@ -0,0 +1,268 @@ +//! Conditional PostgreSQL query-projection removal and repair. + +use super::*; + +impl PostgresEventStore { + pub async fn remove_query_projection( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + ) -> Result<(), PersistenceError> { + self.remove_query_projection_inner(tenant, entity_type, entity_id, None) + .await + .map(|_| ()) + } + + /// Remove a projection only while its exact journal/snapshot source is current. + pub async fn remove_query_projection_if_source( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + source: ProjectionSourceFence<'_>, + ) -> Result { + self.remove_query_projection_inner(tenant, entity_type, entity_id, Some(source)) + .await + } + + /// Clear a dirty marker only while its exact journal/snapshot source is current. + pub async fn clear_query_projection_dirty_if_source( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + source: ProjectionSourceFence<'_>, + ) -> Result { + let mut tx = self.pool().begin().await.map_err(storage_error)?; + lock_key_contract(&mut tx, tenant, entity_type).await?; + let stream_lock_key = event_stream_lock_key(tenant, entity_type, entity_id); + lock_event_stream(&mut tx, &stream_lock_key).await?; + let journal_sequence: i64 = crate::dbm::postgres_query_scalar!( + "SELECT COALESCE(MAX(sequence_nr), 0) FROM events \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_one(&mut *tx) + .await + .map_err(storage_error)?; + let journal_matches = + u64::try_from(journal_sequence).ok() == Some(source.expected_journal_sequence); + let snapshot_matches = + projection_snapshot_source_matches(&mut tx, tenant, entity_type, entity_id, source) + .await?; + if !journal_matches || !snapshot_matches { + tx.commit().await.map_err(storage_error)?; + return Ok(false); + } + clear_query_projection_dirty(&mut tx, tenant, entity_type, entity_id).await?; + tx.commit().await.map_err(storage_error)?; + Ok(true) + } + + async fn remove_query_projection_inner( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + source: Option>, + ) -> Result { + let source_fenced = source.is_some(); + let mut transaction_timer = + PostgresTransactionTimer::start(QUERY_PROJECTION_REMOVE_OPERATION); + let acquire_started = Instant::now(); + let mut conn = match self.pool().acquire().await { + Ok(conn) => { + record_postgres_pool_acquire_duration( + acquire_started.elapsed(), + QUERY_PROJECTION_REMOVE_OPERATION, + "ok", + ); + conn + } + Err(e) => { + record_postgres_pool_acquire_duration( + acquire_started.elapsed(), + QUERY_PROJECTION_REMOVE_OPERATION, + "error", + ); + return Err(storage_error(e)); + } + }; + let begin_started = Instant::now(); + let mut tx = match conn.begin().await { + Ok(tx) => { + record_postgres_transaction_begin_duration( + begin_started.elapsed(), + QUERY_PROJECTION_REMOVE_OPERATION, + "ok", + ); + tx + } + Err(e) => { + record_postgres_transaction_begin_duration( + begin_started.elapsed(), + QUERY_PROJECTION_REMOVE_OPERATION, + "error", + ); + return Err(storage_error(e)); + } + }; + lock_key_contract(&mut tx, tenant, entity_type).await?; + let stream_lock_key = event_stream_lock_key(tenant, entity_type, entity_id); + lock_event_stream(&mut tx, &stream_lock_key).await?; + let journal_sequence: i64 = crate::dbm::postgres_query_scalar!( + "SELECT COALESCE(MAX(sequence_nr), 0) FROM events \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_one(&mut *tx) + .await + .map_err(storage_error)?; + let source_backed = if journal_sequence > 0 { + true + } else { + crate::dbm::postgres_query_scalar!( + "SELECT EXISTS(SELECT 1 FROM snapshots \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3)", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_one(&mut *tx) + .await + .map_err(storage_error)? + }; + if let Some(source) = source { + let journal_matches = + u64::try_from(journal_sequence).ok() == Some(source.expected_journal_sequence); + let snapshot_matches = + projection_snapshot_source_matches(&mut tx, tenant, entity_type, entity_id, source) + .await?; + if !journal_matches || !snapshot_matches { + tx.commit().await.map_err(storage_error)?; + transaction_timer.set_outcome("source_changed"); + return Ok(false); + } + } + let removed_catalog_sequence: Option = crate::dbm::postgres_query_scalar!( + "DELETE FROM entity_catalog \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ + RETURNING sequence_nr", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_optional(&mut *tx) + .await + .map_err(storage_error)?; + let removed_fields = crate::dbm::postgres_query!("DELETE FROM entity_field_index WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3") + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .execute(&mut *tx) + .await + .map_err(storage_error)?; + if removed_catalog_sequence.is_some() || removed_fields.rows_affected() > 0 { + invalidate_key_coverage_for_derived_write( + &mut tx, + tenant, + entity_type, + entity_id, + DerivedWriteSource::Catalog, + ) + .await?; + } + if source_fenced { + clear_query_projection_dirty(&mut tx, tenant, entity_type, entity_id).await?; + } else if source_backed { + mark_query_projection_dirty(&mut tx, tenant, entity_type, entity_id).await?; + } else { + clear_query_projection_dirty(&mut tx, tenant, entity_type, entity_id).await?; + } + let commit_started = Instant::now(); + tx.commit().await.map_err(|e| { + record_postgres_transaction_commit_duration( + commit_started.elapsed(), + QUERY_PROJECTION_REMOVE_OPERATION, + "error", + ); + storage_error(e) + })?; + record_postgres_transaction_commit_duration( + commit_started.elapsed(), + QUERY_PROJECTION_REMOVE_OPERATION, + "ok", + ); + transaction_timer.set_outcome("ok"); + Ok(true) + } + + /// Remove only an exact attempted projection row during unstable-source cleanup. + #[expect( + clippy::too_many_arguments, + reason = "exact projection cleanup boundary" + )] + pub async fn remove_query_projection_if_exact( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + status: &str, + fields: &serde_json::Value, + state: &serde_json::Value, + sequence_nr: u64, + ) -> Result { + let status = canonical_projection_status(status, state); + let sequence_nr = i64::try_from(sequence_nr).map_err(|_| { + PersistenceError::Storage("projection sequence exceeds PostgreSQL bigint".to_string()) + })?; + let mut tx = self.pool().begin().await.map_err(storage_error)?; + lock_key_contract(&mut tx, tenant, entity_type).await?; + let stream_lock_key = event_stream_lock_key(tenant, entity_type, entity_id); + lock_event_stream(&mut tx, &stream_lock_key).await?; + let removed_catalog_sequence: Option = crate::dbm::postgres_query_scalar!( + "DELETE FROM entity_catalog \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ + AND status = $4 AND fields = $5 AND state = $6 AND sequence_nr = $7 \ + RETURNING sequence_nr", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .bind(status) + .bind(fields) + .bind(state) + .bind(sequence_nr) + .fetch_optional(&mut *tx) + .await + .map_err(storage_error)?; + if removed_catalog_sequence.is_some() { + crate::dbm::postgres_query!( + "DELETE FROM entity_field_index \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .execute(&mut *tx) + .await + .map_err(storage_error)?; + invalidate_key_coverage_for_derived_write( + &mut tx, + tenant, + entity_type, + entity_id, + DerivedWriteSource::Catalog, + ) + .await?; + mark_query_projection_dirty(&mut tx, tenant, entity_type, entity_id).await?; + } + tx.commit().await.map_err(storage_error)?; + Ok(removed_catalog_sequence.is_some()) + } +} diff --git a/crates/temper-store-postgres/src/platform/query_projection_write.rs b/crates/temper-store-postgres/src/platform/query_projection_write.rs new file mode 100644 index 000000000..0950bdc91 --- /dev/null +++ b/crates/temper-store-postgres/src/platform/query_projection_write.rs @@ -0,0 +1,446 @@ +//! Source-fenced PostgreSQL query-projection writes. + +use super::*; + +impl PostgresEventStore { + pub async fn upsert_query_projection( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + status: &str, + fields: &serde_json::Value, + sequence_nr: u64, + ) -> Result<(), PersistenceError> { + let state = serde_json::json!({ + "entity_type": entity_type, + "entity_id": entity_id, + "status": status, + "item_count": 0, + "counters": {}, + "booleans": {}, + "lists": {}, + "fields": fields, + "events": [], + "total_event_count": sequence_nr, + "sequence_nr": sequence_nr + }); + self.upsert_query_projection_with_state( + tenant, + entity_type, + entity_id, + status, + fields, + &state, + sequence_nr, + ) + .await + } + + #[expect(clippy::too_many_arguments, reason = "projection upsert boundary")] + pub async fn upsert_query_projection_with_state( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + status: &str, + fields: &serde_json::Value, + state: &serde_json::Value, + sequence_nr: u64, + ) -> Result<(), PersistenceError> { + self.upsert_query_projection_with_state_inner( + tenant, + entity_type, + entity_id, + status, + fields, + state, + sequence_nr, + None, + ) + .await + .map(|_| ()) + } + + /// Upsert a projection only while its exact journal/snapshot source is current. + #[expect( + clippy::too_many_arguments, + reason = "source-fenced projection boundary" + )] + pub async fn upsert_query_projection_with_state_if_source( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + status: &str, + fields: &serde_json::Value, + state: &serde_json::Value, + sequence_nr: u64, + source: ProjectionSourceFence<'_>, + ) -> Result { + self.upsert_query_projection_with_state_inner( + tenant, + entity_type, + entity_id, + status, + fields, + state, + sequence_nr, + Some(source), + ) + .await + } + + #[expect(clippy::too_many_arguments, reason = "projection upsert boundary")] + async fn upsert_query_projection_with_state_inner( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + status: &str, + fields: &serde_json::Value, + state: &serde_json::Value, + sequence_nr: u64, + source: Option>, + ) -> Result { + let source_fenced = source.is_some(); + let status = canonical_projection_status(status, state); + let projection_hash = json_hash(fields); + let (new_index, indexed_fields, skipped_fields) = scalar_index_fields(fields); + let mut transaction_timer = + PostgresTransactionTimer::start(QUERY_PROJECTION_UPSERT_OPERATION); + let acquire_started = Instant::now(); + let mut conn = match self.pool().acquire().await { + Ok(conn) => { + record_postgres_pool_acquire_duration( + acquire_started.elapsed(), + QUERY_PROJECTION_UPSERT_OPERATION, + "ok", + ); + conn + } + Err(e) => { + record_postgres_pool_acquire_duration( + acquire_started.elapsed(), + QUERY_PROJECTION_UPSERT_OPERATION, + "error", + ); + return Err(storage_error(e)); + } + }; + let begin_started = Instant::now(); + let mut tx = match conn.begin().await { + Ok(tx) => { + record_postgres_transaction_begin_duration( + begin_started.elapsed(), + QUERY_PROJECTION_UPSERT_OPERATION, + "ok", + ); + tx + } + Err(e) => { + record_postgres_transaction_begin_duration( + begin_started.elapsed(), + QUERY_PROJECTION_UPSERT_OPERATION, + "error", + ); + return Err(storage_error(e)); + } + }; + + lock_key_contract(&mut tx, tenant, entity_type).await?; + let stream_lock_key = event_stream_lock_key(tenant, entity_type, entity_id); + lock_event_stream(&mut tx, &stream_lock_key).await?; + + let journal_sequence: i64 = crate::dbm::postgres_query_scalar!( + "SELECT COALESCE(MAX(sequence_nr), 0) FROM events \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_one(&mut *tx) + .await + .map_err(storage_error)?; + + let source_backed = if journal_sequence > 0 { + true + } else { + crate::dbm::postgres_query_scalar!( + "SELECT EXISTS(SELECT 1 FROM snapshots \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3)", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_one(&mut *tx) + .await + .map_err(storage_error)? + }; + + if let Some(source) = source { + let journal_matches = + u64::try_from(journal_sequence).ok() == Some(source.expected_journal_sequence); + let snapshot_matches = + projection_snapshot_source_matches(&mut tx, tenant, entity_type, entity_id, source) + .await?; + let source_sequence = if source.expected_journal_sequence > 0 { + source.expected_journal_sequence + } else { + source + .expected_snapshot + .map(|snapshot| snapshot.sequence_nr) + .unwrap_or(0) + }; + if !journal_matches || !snapshot_matches || sequence_nr != source_sequence { + tx.commit().await.map_err(storage_error)?; + transaction_timer.set_outcome("source_changed"); + return Ok(false); + } + } + + let previous_catalog: Option = + crate::dbm::postgres_query_as!( + "SELECT status, projection_hash, sequence_nr \ + FROM entity_catalog \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ + FOR UPDATE", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_optional(&mut *tx) + .await + .map_err(storage_error)?; + + let new_sequence_nr = i64::try_from(sequence_nr).map_err(|_| { + PersistenceError::Storage("projection sequence exceeds PostgreSQL bigint".to_string()) + })?; + let incoming_is_stale = if source_fenced { + false + } else if journal_sequence > 0 { + new_sequence_nr != journal_sequence + } else { + previous_catalog + .as_ref() + .is_some_and(|(_, _, existing_sequence)| *existing_sequence > new_sequence_nr) + }; + let previous_catalog = if incoming_is_stale { + let commit_started = Instant::now(); + tx.commit().await.map_err(|e| { + record_postgres_transaction_commit_duration( + commit_started.elapsed(), + QUERY_PROJECTION_UPSERT_OPERATION, + "error", + ); + storage_error(e) + })?; + record_postgres_transaction_commit_duration( + commit_started.elapsed(), + QUERY_PROJECTION_UPSERT_OPERATION, + "ok", + ); + record_postgres_projection_index_fields( + QUERY_PROJECTION_UPSERT_OPERATION, + entity_type, + indexed_fields, + skipped_fields, + ); + record_postgres_projection_index_reconciliation( + QUERY_PROJECTION_UPSERT_OPERATION, + "stale_skipped", + ); + transaction_timer.set_outcome("stale_skipped"); + return Ok(false); + } else if previous_catalog.is_some() { + update_query_projection_catalog_row( + &mut tx, + QueryProjectionCatalogUpdate { + tenant, + entity_type, + entity_id, + status, + fields, + state, + sequence_nr, + projection_hash: projection_hash.as_str(), + }, + ) + .await?; + previous_catalog + } else { + let inserted: Option = crate::dbm::postgres_query_scalar!( + "INSERT INTO entity_catalog \ + (tenant, entity_type, entity_id, status, fields, state, sequence_nr, projection_version, projection_hash, updated_at) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, 2, $8, now()) \ + ON CONFLICT (tenant, entity_type, entity_id) DO NOTHING \ + RETURNING 1", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .bind(status) + .bind(fields) + .bind(state) + .bind(sequence_nr as i64) + .bind(projection_hash.as_str()) + .fetch_optional(&mut *tx) + .await + .map_err(storage_error)?; + + if inserted.is_some() { + None + } else { + let raced_catalog: Option = + crate::dbm::postgres_query_as!( + "SELECT status, projection_hash, sequence_nr \ + FROM entity_catalog \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ + FOR UPDATE", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_optional(&mut *tx) + .await + .map_err(storage_error)?; + let raced_is_stale = if source_fenced { + false + } else if journal_sequence > 0 { + new_sequence_nr != journal_sequence + } else { + raced_catalog + .as_ref() + .is_some_and(|(_, _, existing_sequence)| { + *existing_sequence > new_sequence_nr + }) + }; + if raced_is_stale { + let commit_started = Instant::now(); + tx.commit().await.map_err(|e| { + record_postgres_transaction_commit_duration( + commit_started.elapsed(), + QUERY_PROJECTION_UPSERT_OPERATION, + "error", + ); + storage_error(e) + })?; + record_postgres_transaction_commit_duration( + commit_started.elapsed(), + QUERY_PROJECTION_UPSERT_OPERATION, + "ok", + ); + record_postgres_projection_index_fields( + QUERY_PROJECTION_UPSERT_OPERATION, + entity_type, + indexed_fields, + skipped_fields, + ); + record_postgres_projection_index_reconciliation( + QUERY_PROJECTION_UPSERT_OPERATION, + "stale_skipped", + ); + transaction_timer.set_outcome("stale_skipped"); + return Ok(false); + } + update_query_projection_catalog_row( + &mut tx, + QueryProjectionCatalogUpdate { + tenant, + entity_type, + entity_id, + status, + fields, + state, + sequence_nr, + projection_hash: projection_hash.as_str(), + }, + ) + .await?; + raced_catalog + } + }; + + let catalog_changed = + previous_catalog + .as_ref() + .is_none_or(|(old_status, old_hash, old_sequence)| { + old_status.as_str() != status + || old_hash.as_str() != projection_hash.as_str() + || *old_sequence != new_sequence_nr + }); + if catalog_changed { + invalidate_key_coverage_for_derived_write( + &mut tx, + tenant, + entity_type, + entity_id, + DerivedWriteSource::Catalog, + ) + .await?; + } + + let should_reconcile_index = + previous_catalog + .as_ref() + .is_none_or(|(old_status, old_hash, _)| { + old_status.as_str() != status || old_hash.as_str() != projection_hash.as_str() + }); + let reconciliation_path = if should_reconcile_index { + if previous_catalog.is_some() { + "diff" + } else { + "insert" + } + } else { + "skipped_unchanged" + }; + + if should_reconcile_index { + reconcile_query_projection_field_index( + &mut tx, + tenant, + entity_type, + entity_id, + status, + &new_index, + ) + .await?; + } + + if source_fenced { + clear_query_projection_dirty(&mut tx, tenant, entity_type, entity_id).await?; + } else if source_backed { + mark_query_projection_dirty(&mut tx, tenant, entity_type, entity_id).await?; + } else { + clear_query_projection_dirty(&mut tx, tenant, entity_type, entity_id).await?; + } + + let commit_started = Instant::now(); + tx.commit().await.map_err(|e| { + record_postgres_transaction_commit_duration( + commit_started.elapsed(), + QUERY_PROJECTION_UPSERT_OPERATION, + "error", + ); + storage_error(e) + })?; + record_postgres_transaction_commit_duration( + commit_started.elapsed(), + QUERY_PROJECTION_UPSERT_OPERATION, + "ok", + ); + record_postgres_projection_index_fields( + QUERY_PROJECTION_UPSERT_OPERATION, + entity_type, + indexed_fields, + skipped_fields, + ); + record_postgres_projection_index_reconciliation( + QUERY_PROJECTION_UPSERT_OPERATION, + reconciliation_path, + ); + transaction_timer.set_outcome("ok"); + Ok(true) + } +} diff --git a/crates/temper-store-postgres/src/platform/spec_publication.rs b/crates/temper-store-postgres/src/platform/spec_publication.rs new file mode 100644 index 000000000..8bf33c5e2 --- /dev/null +++ b/crates/temper-store-postgres/src/platform/spec_publication.rs @@ -0,0 +1,247 @@ +//! Atomic PostgreSQL tenant spec publication. + +use super::*; + +impl PostgresEventStore { + /// Atomically publish changed specs and remove replace-mode omissions. + #[expect( + clippy::too_many_arguments, + reason = "atomic publication boundary mirrors the platform-store contract" + )] + pub async fn publish_specs( + &self, + tenant: &str, + specs: &[(&str, &str, &str, &str)], + replace: bool, + constraints: Option>, + policy: Option<&str>, + os_app: Option<&PostgresInstalledAppRow>, + wasm_modules: &[(&str, &[u8], &str, &str)], + policy_owner: Option<&str>, + policy_entries: &[(&str, &str, &str)], + ) -> Result, PersistenceError> { + let mut seen_types = std::collections::BTreeSet::new(); + for (entity_type, _, _, _) in specs { + if !seen_types.insert(*entity_type) { + return Err(PersistenceError::Storage(format!( + "duplicate entity type {entity_type} in spec publication for tenant {tenant}" + ))); + } + } + let mut seen_modules = std::collections::BTreeSet::new(); + for (module_name, _, _, _) in wasm_modules { + if !seen_modules.insert(*module_name) { + return Err(PersistenceError::Storage(format!( + "duplicate WASM module {module_name} in spec publication for tenant {tenant}" + ))); + } + } + let mut seen_policy_ids = std::collections::BTreeSet::new(); + for (policy_id, _, _) in policy_entries { + if !seen_policy_ids.insert(*policy_id) { + return Err(PersistenceError::Storage(format!( + "duplicate Cedar policy {policy_id} in spec publication for tenant {tenant}" + ))); + } + } + let mut tx = self.pool().begin().await.map_err(storage_error)?; + sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))") + .bind(format!("spec-publication:{tenant}")) + .execute(&mut *tx) + .await + .map_err(storage_error)?; + let incoming_types = specs + .iter() + .map(|(entity_type, _, _, _)| *entity_type) + .collect::>(); + let removed_entity_types = if replace { + crate::dbm::postgres_query_scalar!( + "SELECT entity_type FROM specs WHERE tenant = $1 FOR UPDATE", + ) + .bind(tenant) + .fetch_all(&mut *tx) + .await + .map_err(storage_error)? + .into_iter() + .filter(|entity_type: &String| !incoming_types.contains(entity_type.as_str())) + .collect::>() + } else { + Vec::new() + }; + for (entity_type, ioa_source, csdl_xml, content_hash) in specs { + crate::dbm::postgres_query!( + "INSERT INTO specs \ + (tenant, entity_type, ioa_source, csdl_xml, content_hash, committed, version, verified, verification_status, updated_at) \ + VALUES ($1, $2, $3, $4, $5, true, 1, false, 'pending', now()) \ + ON CONFLICT (tenant, entity_type) DO UPDATE SET \ + ioa_source = CASE WHEN specs.content_hash IS DISTINCT FROM EXCLUDED.content_hash OR specs.csdl_xml IS DISTINCT FROM EXCLUDED.csdl_xml THEN EXCLUDED.ioa_source ELSE specs.ioa_source END, \ + csdl_xml = CASE WHEN specs.content_hash IS DISTINCT FROM EXCLUDED.content_hash OR specs.csdl_xml IS DISTINCT FROM EXCLUDED.csdl_xml THEN EXCLUDED.csdl_xml ELSE specs.csdl_xml END, \ + content_hash = CASE WHEN specs.content_hash IS DISTINCT FROM EXCLUDED.content_hash OR specs.csdl_xml IS DISTINCT FROM EXCLUDED.csdl_xml THEN EXCLUDED.content_hash ELSE specs.content_hash END, \ + committed = true, \ + version = CASE WHEN specs.content_hash IS DISTINCT FROM EXCLUDED.content_hash OR specs.csdl_xml IS DISTINCT FROM EXCLUDED.csdl_xml THEN specs.version + 1 ELSE specs.version END, \ + verified = CASE WHEN specs.content_hash IS DISTINCT FROM EXCLUDED.content_hash OR specs.csdl_xml IS DISTINCT FROM EXCLUDED.csdl_xml THEN false ELSE specs.verified END, \ + verification_status = CASE WHEN specs.content_hash IS DISTINCT FROM EXCLUDED.content_hash OR specs.csdl_xml IS DISTINCT FROM EXCLUDED.csdl_xml THEN 'pending' ELSE specs.verification_status END, \ + levels_passed = CASE WHEN specs.content_hash IS DISTINCT FROM EXCLUDED.content_hash OR specs.csdl_xml IS DISTINCT FROM EXCLUDED.csdl_xml THEN NULL ELSE specs.levels_passed END, \ + levels_total = CASE WHEN specs.content_hash IS DISTINCT FROM EXCLUDED.content_hash OR specs.csdl_xml IS DISTINCT FROM EXCLUDED.csdl_xml THEN NULL ELSE specs.levels_total END, \ + verification_result = CASE WHEN specs.content_hash IS DISTINCT FROM EXCLUDED.content_hash OR specs.csdl_xml IS DISTINCT FROM EXCLUDED.csdl_xml THEN NULL ELSE specs.verification_result END, \ + updated_at = CASE WHEN specs.content_hash IS DISTINCT FROM EXCLUDED.content_hash OR specs.csdl_xml IS DISTINCT FROM EXCLUDED.csdl_xml OR specs.committed = false THEN now() ELSE specs.updated_at END", + ) + .bind(tenant) + .bind(*entity_type) + .bind(*ioa_source) + .bind(*csdl_xml) + .bind(*content_hash) + .execute(&mut *tx) + .await + .map_err(storage_error)?; + } + for entity_type in &removed_entity_types { + crate::dbm::postgres_query!( + "DELETE FROM specs WHERE tenant = $1 AND entity_type = $2", + ) + .bind(tenant) + .bind(entity_type) + .execute(&mut *tx) + .await + .map_err(storage_error)?; + } + match constraints { + None => {} + Some(Some(source)) => { + crate::dbm::postgres_query!( + "INSERT INTO tenant_constraints \ + (tenant, cross_invariants_toml, version, updated_at) \ + VALUES ($1, $2, 1, now()) \ + ON CONFLICT (tenant) DO UPDATE SET \ + cross_invariants_toml = EXCLUDED.cross_invariants_toml, \ + version = tenant_constraints.version + 1, updated_at = now()", + ) + .bind(tenant) + .bind(source) + .execute(&mut *tx) + .await + .map_err(storage_error)?; + } + Some(None) => { + crate::dbm::postgres_query!("DELETE FROM tenant_constraints WHERE tenant = $1",) + .bind(tenant) + .execute(&mut *tx) + .await + .map_err(storage_error)?; + } + } + if let Some(policy) = policy { + crate::dbm::postgres_query!( + "INSERT INTO tenant_policies (tenant, policy_text, updated_at) \ + VALUES ($1, $2, now()) \ + ON CONFLICT (tenant) DO UPDATE SET \ + policy_text = EXCLUDED.policy_text, updated_at = now()", + ) + .bind(tenant) + .bind(policy) + .execute(&mut *tx) + .await + .map_err(storage_error)?; + } + if let Some(owner) = policy_owner { + crate::dbm::postgres_query!( + "DELETE FROM policies WHERE tenant = $1 AND created_by = $2", + ) + .bind(tenant) + .bind(owner) + .execute(&mut *tx) + .await + .map_err(storage_error)?; + for (policy_id, cedar_text, created_by) in policy_entries { + let policy_hash = compute_policy_hash(cedar_text); + crate::dbm::postgres_query!( + "INSERT INTO policies \ + (tenant, policy_id, cedar_text, policy_hash, created_at, created_by, enabled) \ + VALUES ($1, $2, $3, $4, now(), $5, true) \ + ON CONFLICT (tenant, policy_id) DO UPDATE SET \ + cedar_text = EXCLUDED.cedar_text, policy_hash = EXCLUDED.policy_hash, \ + created_by = EXCLUDED.created_by, created_at = now(), enabled = true", + ) + .bind(tenant) + .bind(*policy_id) + .bind(*cedar_text) + .bind(policy_hash) + .bind(*created_by) + .execute(&mut *tx) + .await + .map_err(storage_error)?; + } + } + for (module_name, wasm_bytes, sha256_hash, source) in wasm_modules { + let replace_uploaded_wasm = *source == BUNDLED_REPLACE_UPLOAD_SOURCE; + let persisted_source = if replace_uploaded_wasm { + "bundled" + } else { + *source + }; + crate::dbm::postgres_query!( + "INSERT INTO wasm_modules \ + (tenant, module_name, wasm_bytes, sha256_hash, version, size_bytes, updated_at, source) \ + VALUES ($1, $2, $3, $4, 1, $5, now(), $6) \ + ON CONFLICT (tenant, module_name) DO UPDATE SET \ + wasm_bytes = EXCLUDED.wasm_bytes, sha256_hash = EXCLUDED.sha256_hash, \ + version = wasm_modules.version + 1, size_bytes = EXCLUDED.size_bytes, \ + updated_at = now(), source = EXCLUDED.source \ + WHERE wasm_modules.sha256_hash IS DISTINCT FROM EXCLUDED.sha256_hash \ + AND ($7 OR EXCLUDED.source = 'upload' OR wasm_modules.source = 'bundled')", + ) + .bind(tenant) + .bind(*module_name) + .bind(*wasm_bytes) + .bind(*sha256_hash) + .bind(wasm_bytes.len() as i32) + .bind(persisted_source) + .bind(replace_uploaded_wasm) + .execute(&mut *tx) + .await + .map_err(storage_error)?; + } + if let Some(record) = os_app { + crate::dbm::postgres_query!( + "INSERT INTO tenant_installed_apps \ + (tenant, app_name, source_kind, app_ref, version_hash, pinned_version_hash, current_version_hash, follow_policy, closure_id, registry_url, registry_tenant, \ + app_version, bundle_digest, spec_digest, policy_digest, wasm_digest, content_digest, seed_digest, installed_at, last_reconciled_at, status) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, now(), now(), $19) \ + ON CONFLICT (tenant, app_name) DO UPDATE SET \ + source_kind = EXCLUDED.source_kind, app_ref = EXCLUDED.app_ref, \ + version_hash = EXCLUDED.version_hash, pinned_version_hash = EXCLUDED.pinned_version_hash, \ + current_version_hash = EXCLUDED.current_version_hash, follow_policy = EXCLUDED.follow_policy, \ + closure_id = EXCLUDED.closure_id, registry_url = EXCLUDED.registry_url, \ + registry_tenant = EXCLUDED.registry_tenant, app_version = EXCLUDED.app_version, \ + bundle_digest = EXCLUDED.bundle_digest, spec_digest = EXCLUDED.spec_digest, \ + policy_digest = EXCLUDED.policy_digest, wasm_digest = EXCLUDED.wasm_digest, \ + content_digest = EXCLUDED.content_digest, seed_digest = EXCLUDED.seed_digest, \ + last_reconciled_at = now(), status = EXCLUDED.status", + ) + .bind(tenant) + .bind(&record.app_name) + .bind(&record.source_kind) + .bind(&record.app_ref) + .bind(&record.version_hash) + .bind(&record.pinned_version_hash) + .bind(&record.current_version_hash) + .bind(&record.follow_policy) + .bind(&record.closure_id) + .bind(&record.registry_url) + .bind(&record.registry_tenant) + .bind(&record.app_version) + .bind(&record.bundle_digest) + .bind(&record.spec_digest) + .bind(&record.policy_digest) + .bind(&record.wasm_digest) + .bind(&record.content_digest) + .bind(&record.seed_digest) + .bind(&record.status) + .execute(&mut *tx) + .await + .map_err(storage_error)?; + } + tx.commit().await.map_err(storage_error)?; + Ok(removed_entity_types) + } +} diff --git a/crates/temper-store-postgres/src/schema.rs b/crates/temper-store-postgres/src/schema.rs index 865a036ab..f3df2f1ef 100644 --- a/crates/temper-store-postgres/src/schema.rs +++ b/crates/temper-store-postgres/src/schema.rs @@ -362,6 +362,21 @@ pub const CREATE_ENTITY_FIELD_INDEX_STATUS: &str = "\ CREATE INDEX IF NOT EXISTS idx_efi_status ON entity_field_index (tenant, entity_type, status);"; +/// Entities whose journal/snapshot source is newer than their query projection. +pub const CREATE_QUERY_PROJECTION_DIRTY_TABLE: &str = "\ +CREATE TABLE IF NOT EXISTS query_projection_dirty ( + tenant TEXT NOT NULL, + entity_type TEXT NOT NULL, + entity_id TEXT NOT NULL, + updated_at TIMESTAMPTZ NOT NULL DEFAULT now(), + PRIMARY KEY (tenant, entity_type, entity_id) +);"; + +/// Bounded dirty-projection enumeration by tenant and entity type. +pub const CREATE_QUERY_PROJECTION_DIRTY_TYPE_INDEX: &str = "\ +CREATE INDEX IF NOT EXISTS idx_query_projection_dirty_type + ON query_projection_dirty (tenant, entity_type, entity_id);"; + /// Feature request records generated from trajectory analysis. pub const CREATE_FEATURE_REQUESTS_TABLE: &str = "\ CREATE TABLE IF NOT EXISTS feature_requests ( diff --git a/crates/temper-store-postgres/src/segments.rs b/crates/temper-store-postgres/src/segments.rs index a004dfc59..74c88d1b7 100644 --- a/crates/temper-store-postgres/src/segments.rs +++ b/crates/temper-store-postgres/src/segments.rs @@ -1,6 +1,50 @@ use sqlx::PgConnection; use temper_runtime::persistence::PersistenceError; +async fn rebuild_open_segment_from_journal( + conn: &mut PgConnection, + tenant: &str, + entity_type: &str, + entity_id: &str, + current_seq: u64, +) -> Result { + sqlx::query( + "UPDATE events SET segment_index = 0 \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .execute(&mut *conn) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + sqlx::query( + "DELETE FROM event_segments \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .execute(&mut *conn) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + sqlx::query( + "INSERT INTO event_segments \ + (tenant, entity_type, entity_id, segment_index, start_sequence_nr, end_sequence_nr, event_count) \ + VALUES ($1, $2, $3, 0, 1, $4, $4)", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .bind(i64::try_from(current_seq).map_err(|_| { + PersistenceError::Storage("journal sequence exceeds PostgreSQL bigint".to_string()) + })?) + .execute(conn) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + Ok(0) +} + pub(crate) async fn open_segment_for_append( conn: &mut PgConnection, tenant: &str, @@ -8,8 +52,34 @@ pub(crate) async fn open_segment_for_append( entity_id: &str, current_seq: u64, ) -> Result { - let segment_row: Option<(i64,)> = sqlx::query_as( - "SELECT segment_index FROM event_segments \ + if current_seq == 0 { + // Snapshot-only generations do not own journal segments. Remove legacy + // metadata before assigning the first real event to segment zero. + sqlx::query( + "DELETE FROM event_segments \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .execute(&mut *conn) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + sqlx::query( + "INSERT INTO event_segments \ + (tenant, entity_type, entity_id, segment_index, start_sequence_nr) \ + VALUES ($1, $2, $3, 0, 1)", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .execute(conn) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + return Ok(0); + } + let segment_row: Option<(i64, i64, Option)> = sqlx::query_as( + "SELECT segment_index, start_sequence_nr, end_sequence_nr FROM event_segments \ WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 AND sealed_at IS NULL \ ORDER BY segment_index DESC LIMIT 1", ) @@ -21,10 +91,19 @@ pub(crate) async fn open_segment_for_append( .map_err(|e| PersistenceError::Storage(e.to_string()))?; match segment_row { - Some((idx,)) => Ok(idx), + Some((idx, start, end)) + if start <= current_seq.saturating_add(1) as i64 + && end.is_none_or(|end| end >= start) => + { + Ok(idx) + } + Some(_) => { + rebuild_open_segment_from_journal(conn, tenant, entity_type, entity_id, current_seq) + .await + } None => { - let row: (i64,) = sqlx::query_as( - "SELECT COALESCE(MAX(segment_index), 0) FROM events \ + let segment_count: (i64,) = sqlx::query_as( + "SELECT COUNT(*) FROM event_segments \ WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", ) .bind(tenant) @@ -32,6 +111,30 @@ pub(crate) async fn open_segment_for_append( .bind(entity_id) .fetch_one(&mut *conn) .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + if segment_count.0 == 0 { + return rebuild_open_segment_from_journal( + conn, + tenant, + entity_type, + entity_id, + current_seq, + ) + .await; + } + let row: (i64,) = sqlx::query_as( + "SELECT GREATEST(\ + COALESCE((SELECT MAX(segment_index) FROM events \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3), -1), \ + COALESCE((SELECT MAX(segment_index) FROM event_segments \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3), -1)\ + ) + 1", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_one(&mut *conn) + .await .map_err(|e| PersistenceError::Storage(e.to_string()))?; sqlx::query( "INSERT INTO event_segments \ @@ -83,30 +186,96 @@ pub(crate) async fn rotate_after_snapshot( entity_id: &str, sequence_nr: u64, ) -> Result<(), PersistenceError> { + let boundary = i64::try_from(sequence_nr).map_err(|_| { + PersistenceError::Storage("snapshot sequence exceeds PostgreSQL bigint".to_string()) + })?; + let journal: (i64,) = sqlx::query_as( + "SELECT COALESCE(MAX(sequence_nr), 0) FROM events \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_one(&mut *conn) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + if journal.0 == 0 { + sqlx::query( + "DELETE FROM event_segments \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .execute(conn) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + return Ok(()); + } + if boundary == 0 || boundary > journal.0 { + // A snapshot is a recovery accelerator, not a journal boundary. A + // migration snapshot ahead of the durable journal must not manufacture + // sealed ranges or an open segment beyond the journal high-water. + return Ok(()); + } let row: (i64,) = sqlx::query_as( - "SELECT COALESCE(MAX(segment_index), 0) FROM events \ - WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 AND sequence_nr <= $4", + "SELECT COALESCE(\ + (SELECT MAX(segment_index) FROM events \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 AND sequence_nr <= $4), \ + (SELECT MAX(segment_index) FROM event_segments \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3), \ + 0)", ) .bind(tenant) .bind(entity_type) .bind(entity_id) - .bind(sequence_nr as i64) + .bind(boundary) .fetch_one(&mut *conn) .await .map_err(|e| PersistenceError::Storage(e.to_string()))?; let current_segment = row.0; + let current_start: (i64,) = sqlx::query_as( + "SELECT COALESCE(\ + (SELECT start_sequence_nr FROM event_segments \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 AND segment_index = $4), \ + (SELECT MIN(sequence_nr) FROM events \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 AND segment_index = $4), \ + 1)", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .bind(current_segment) + .fetch_one(&mut *conn) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + let current_count: (i64,) = sqlx::query_as( + "SELECT COUNT(*) FROM events \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ + AND segment_index = $4 AND sequence_nr <= $5", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .bind(current_segment) + .bind(boundary) + .fetch_one(&mut *conn) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; sqlx::query( "INSERT INTO event_segments \ (tenant, entity_type, entity_id, segment_index, start_sequence_nr, end_sequence_nr, snapshot_sequence, event_count, sealed_at) \ - VALUES ($1, $2, $3, $4, 1, $5, $5, $5, now()) \ + VALUES ($1, $2, $3, $4, $5, $6, $6, $7, now()) \ ON CONFLICT (tenant, entity_type, entity_id, segment_index) DO NOTHING", ) .bind(tenant) .bind(entity_type) .bind(entity_id) .bind(current_segment) - .bind(sequence_nr as i64) + .bind(current_start.0) + .bind(boundary) + .bind(current_count.0) .execute(&mut *conn) .await .map_err(|e| PersistenceError::Storage(e.to_string()))?; @@ -114,29 +283,71 @@ pub(crate) async fn rotate_after_snapshot( sqlx::query( "UPDATE event_segments \ SET end_sequence_nr = $5, snapshot_sequence = $5, sealed_at = now(), \ - event_count = GREATEST($5 - start_sequence_nr + 1, 0) \ + event_count = $6 \ WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 AND segment_index = $4", ) .bind(tenant) .bind(entity_type) .bind(entity_id) .bind(current_segment) - .bind(sequence_nr as i64) + .bind(boundary) + .bind(current_count.0) .execute(&mut *conn) .await .map_err(|e| PersistenceError::Storage(e.to_string()))?; + let next_segment = current_segment + .checked_add(1) + .ok_or_else(|| PersistenceError::Storage("event segment index overflow".to_string()))?; + sqlx::query( + "UPDATE events SET segment_index = $5 \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ + AND segment_index = $4 AND sequence_nr > $6", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .bind(current_segment) + .bind(next_segment) + .bind(boundary) + .execute(&mut *conn) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + + let tail: (i64, Option) = sqlx::query_as( + "SELECT COUNT(*), MAX(sequence_nr) FROM events \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ + AND segment_index = $4 AND sequence_nr > $5", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .bind(next_segment) + .bind(boundary) + .fetch_one(&mut *conn) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + let next_start = boundary.checked_add(1).ok_or_else(|| { + PersistenceError::Storage("snapshot successor sequence overflow".to_string()) + })?; sqlx::query( "INSERT INTO event_segments \ - (tenant, entity_type, entity_id, segment_index, start_sequence_nr) \ - VALUES ($1, $2, $3, $4, $5) \ - ON CONFLICT (tenant, entity_type, entity_id, segment_index) DO NOTHING", + (tenant, entity_type, entity_id, segment_index, start_sequence_nr, end_sequence_nr, snapshot_sequence, event_count, sealed_at) \ + VALUES ($1, $2, $3, $4, $5, $6, NULL, $7, NULL) \ + ON CONFLICT (tenant, entity_type, entity_id, segment_index) DO UPDATE SET \ + start_sequence_nr = EXCLUDED.start_sequence_nr, \ + end_sequence_nr = EXCLUDED.end_sequence_nr, \ + snapshot_sequence = NULL, \ + event_count = EXCLUDED.event_count, \ + sealed_at = NULL", ) .bind(tenant) .bind(entity_type) .bind(entity_id) - .bind(current_segment + 1) - .bind(sequence_nr as i64 + 1) + .bind(next_segment) + .bind(next_start) + .bind(tail.1) + .bind(tail.0) .execute(conn) .await .map_err(|e| PersistenceError::Storage(e.to_string()))?; diff --git a/crates/temper-store-postgres/src/store.rs b/crates/temper-store-postgres/src/store.rs index 21aa5c717..524357f6d 100644 --- a/crates/temper-store-postgres/src/store.rs +++ b/crates/temper-store-postgres/src/store.rs @@ -4,12 +4,23 @@ //! `UNIQUE (entity_type, entity_id, sequence_nr)` constraint to enforce //! optimistic concurrency on appends. +mod append; +mod append_batch; +mod key_index; +mod snapshot; + +#[cfg(test)] +mod tests; + use std::time::Instant; -use sqlx::{Acquire, PgPool}; +use sqlx::{Acquire, PgConnection, PgPool}; use temper_runtime::persistence::{ - EntityVectorCandidate, EntityVectorRow, EventMetadata, EventStore, PersistenceAppend, - PersistenceAppendResult, PersistenceEnvelope, PersistenceError, pack_f32_le, unpack_f32_le, + EntityKeyLookup, EntityVectorCandidate, EntityVectorRow, EventMetadata, EventStore, + IndexReconciliation, JournalBoundary, KeyContractActivation, PersistenceAppend, + PersistenceAppendResult, PersistenceBatchIdempotency, PersistenceEnvelope, PersistenceError, + SnapshotSourceFence, is_state_materialization_event_for, is_state_materialization_payload_for, + pack_f32_le, unpack_f32_le, }; use temper_runtime::tenant::parse_persistence_id_parts; @@ -19,8 +30,101 @@ use crate::metrics::{ }; use crate::segments; +pub(crate) use key_index::{ + DerivedWriteSource, KeyContractUse, event_stream_lock_key, + invalidate_key_coverage_for_derived_write, invalidate_key_coverage_for_unreconciled_append, + lock_event_stream, lock_key_contract, reconcile_key_contract_state, +}; + const EVENT_APPEND_OPERATION: &str = "event_append"; +fn snapshot_source_matches( + current: Option<&(i64, Vec)>, + expected: &SnapshotSourceFence, +) -> bool { + match expected { + SnapshotSourceFence::Unchecked => true, + SnapshotSourceFence::Absent => current.is_none(), + SnapshotSourceFence::Exact { sequence_nr, state } => current.is_some_and(|current| { + u64::try_from(current.0).ok() == Some(*sequence_nr) + && current.1.as_slice() == state.as_slice() + }), + } +} + +fn append_retires_snapshot( + expected_sequence: u64, + events: &[PersistenceEnvelope], + snapshot_source: &SnapshotSourceFence, + entity_type: &str, + entity_id: &str, +) -> bool { + expected_sequence == 0 + && matches!(snapshot_source, SnapshotSourceFence::Exact { .. }) + && events + .first() + .is_some_and(|event| is_state_materialization_event_for(event, entity_type, entity_id)) +} + +async fn load_snapshot_for_update( + conn: &mut PgConnection, + tenant: &str, + entity_type: &str, + entity_id: &str, +) -> Result)>, PersistenceError> { + crate::dbm::postgres_query_as!( + "SELECT sequence_nr, state FROM snapshots \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ + FOR UPDATE", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_optional(conn) + .await + .map_err(|error| PersistenceError::Storage(error.to_string())) +} + +pub(crate) async fn mark_query_projection_dirty( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + tenant: &str, + entity_type: &str, + entity_id: &str, +) -> Result<(), PersistenceError> { + crate::dbm::postgres_query!( + "INSERT INTO query_projection_dirty (tenant, entity_type, entity_id, updated_at) \ + VALUES ($1, $2, $3, now()) \ + ON CONFLICT (tenant, entity_type, entity_id) \ + DO UPDATE SET updated_at = now()", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .execute(&mut **tx) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + Ok(()) +} + +pub(crate) async fn clear_query_projection_dirty( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + tenant: &str, + entity_type: &str, + entity_id: &str, +) -> Result<(), PersistenceError> { + crate::dbm::postgres_query!( + "DELETE FROM query_projection_dirty \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .execute(&mut **tx) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + Ok(()) +} + /// A PostgreSQL-backed event store. /// /// Persistence IDs follow `"tenant:entity_type:entity_id"` (with legacy @@ -48,6 +152,35 @@ impl PostgresEventStore { // --------------------------------------------------------------------------- impl EventStore for PostgresEventStore { + fn supports_authoritative_key_index(&self) -> bool { + true + } + + async fn batch_idempotency_committed( + &self, + claim: &PersistenceBatchIdempotency, + ) -> Result { + let existing: Option<(String,)> = crate::dbm::postgres_query_as!( + "SELECT intent_hash FROM persistence_batch_idempotency \ + WHERE persistence_id = $1 AND idempotency_key = $2", + ) + .bind(&claim.persistence_id) + .bind(&claim.idempotency_key) + .fetch_optional(&self.pool) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + let Some((committed_hash,)) = existing else { + return Ok(false); + }; + if committed_hash != claim.intent_hash { + return Err(PersistenceError::Storage(format!( + "atomic batch idempotency key '{}' was reused with a different intent", + claim.idempotency_key + ))); + } + Ok(true) + } + /// Append one or more events to the journal. /// /// Events are inserted with consecutive sequence numbers starting from @@ -63,8 +196,15 @@ impl EventStore for PostgresEventStore { expected_sequence: u64, events: &[PersistenceEnvelope], ) -> Result { - self.append_with_index_rows(persistence_id, expected_sequence, events, &[], &[], false) - .await + self.append_with_index_rows( + persistence_id, + expected_sequence, + events, + &[], + &[], + IndexReconciliation::default(), + ) + .await } async fn append_with_index_rows( @@ -74,230 +214,17 @@ impl EventStore for PostgresEventStore { events: &[PersistenceEnvelope], key_rows: &[temper_runtime::persistence::EntityKeyRow], vector_rows: &[EntityVectorRow], - reconcile_vectors: bool, + reconciliation: IndexReconciliation, ) -> Result { - let (tenant, entity_type, entity_id) = - parse_persistence_id_parts(persistence_id).map_err(PersistenceError::Storage)?; - - let mut transaction_timer = PostgresTransactionTimer::start(EVENT_APPEND_OPERATION); - let acquire_started = Instant::now(); - let mut conn = match self.pool.acquire().await { - Ok(conn) => { - record_postgres_pool_acquire_duration( - acquire_started.elapsed(), - EVENT_APPEND_OPERATION, - "ok", - ); - conn - } - Err(e) => { - record_postgres_pool_acquire_duration( - acquire_started.elapsed(), - EVENT_APPEND_OPERATION, - "error", - ); - return Err(PersistenceError::Storage(e.to_string())); - } - }; - let begin_started = Instant::now(); - let mut tx = match conn.begin().await { - Ok(tx) => { - record_postgres_transaction_begin_duration( - begin_started.elapsed(), - EVENT_APPEND_OPERATION, - "ok", - ); - tx - } - Err(e) => { - record_postgres_transaction_begin_duration( - begin_started.elapsed(), - EVENT_APPEND_OPERATION, - "error", - ); - return Err(PersistenceError::Storage(e.to_string())); - } - }; - - let row: Option<(i64,)> = crate::dbm::postgres_query_as!( - "SELECT COALESCE(MAX(sequence_nr), 0) FROM events \ - WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + self.append_with_index_rows_inner( + persistence_id, + expected_sequence, + events, + key_rows, + vector_rows, + reconciliation, ) - .bind(tenant) - .bind(entity_type) - .bind(entity_id) - .fetch_optional(&mut *tx) .await - .map_err(|e| PersistenceError::Storage(e.to_string()))?; - - let current_seq = row.map(|r| r.0 as u64).unwrap_or(0); - if current_seq != expected_sequence { - transaction_timer.set_outcome("concurrency_violation"); - return Err(PersistenceError::ConcurrencyViolation { - expected: expected_sequence, - actual: current_seq, - }); - } - - // ADR-0153: validate declared-key uniqueness BEFORE writing the journal so - // a reject is atomic (the journal does not advance). A different entity - // already holding the key is the violation (reject + surface). - for key in key_rows { - let holder: Option<(String,)> = crate::dbm::postgres_query_as!( - "SELECT entity_id FROM entity_key_index \ - WHERE tenant = $1 AND entity_type = $2 AND key_name = $3 AND key_hash = $4", - ) - .bind(tenant) - .bind(entity_type) - .bind(&key.key_name) - .bind(&key.key_hash) - .fetch_optional(&mut *tx) - .await - .map_err(|e| PersistenceError::Storage(e.to_string()))?; - if let Some((existing,)) = holder - && existing != entity_id - { - return Err(PersistenceError::Storage(format!( - "duplicate declared key '{}' for {entity_type}: held by {existing}", - key.key_name - ))); - } - } - - let segment_index = - segments::open_segment_for_append(&mut tx, tenant, entity_type, entity_id, current_seq) - .await?; - - let mut new_seq = expected_sequence; - for event in events { - new_seq += 1; - let metadata_json = serde_json::to_value(&event.metadata) - .map_err(|e| PersistenceError::Serialization(e.to_string()))?; - - if let Err(e) = crate::dbm::postgres_query!( - "INSERT INTO events (tenant, entity_type, entity_id, sequence_nr, segment_index, event_type, payload, metadata) \ - VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", - ) - .bind(tenant) - .bind(entity_type) - .bind(entity_id) - .bind(new_seq as i64) - .bind(segment_index) - .bind(&event.event_type) - .bind(&event.payload) - .bind(metadata_json) - .execute(&mut *tx) - .await - { - let msg = e.to_string(); - if msg.contains("unique") || msg.contains("duplicate key") { - transaction_timer.set_outcome("concurrency_violation"); - return Err(PersistenceError::ConcurrencyViolation { - expected: expected_sequence, - actual: new_seq, - }); - } else { - return Err(PersistenceError::Storage(msg)); - } - } - } - - if new_seq > expected_sequence { - segments::update_segment_after_append( - &mut tx, - tenant, - entity_type, - entity_id, - segment_index, - new_seq, - ) - .await?; - } - - // ADR-0153: co-commit the declared key-index rows in THIS transaction - // (uniqueness was validated above). Drop the entity's prior row for each - // key_name (the value may have changed), then claim the new key_hash. - for key in key_rows { - crate::dbm::postgres_query!( - "DELETE FROM entity_key_index \ - WHERE tenant = $1 AND entity_type = $2 AND key_name = $3 AND entity_id = $4", - ) - .bind(tenant) - .bind(entity_type) - .bind(&key.key_name) - .bind(entity_id) - .execute(&mut *tx) - .await - .map_err(|e| PersistenceError::Storage(e.to_string()))?; - crate::dbm::postgres_query!( - "INSERT INTO entity_key_index \ - (tenant, entity_type, key_name, key_hash, entity_id, sequence_nr) \ - VALUES ($1, $2, $3, $4, $5, $6)", - ) - .bind(tenant) - .bind(entity_type) - .bind(&key.key_name) - .bind(&key.key_hash) - .bind(entity_id) - .bind(new_seq as i64) - .execute(&mut *tx) - .await - .map_err(|e| PersistenceError::Storage(e.to_string()))?; - } - - // ADR-0155: co-commit the derived vector-index rows in THIS transaction. - // When the entity's type declares vector paths (`reconcile_vectors`), DELETE - // all of the entity's rows first, then insert the current ones — so a delete - // transition or a cleared vector/model property (empty `vector_rows`) purges - // the stale rows instead of leaving them to rank forever. No uniqueness - // constraint; vectors are derived ranking state. - if reconcile_vectors { - crate::dbm::postgres_query!( - "DELETE FROM entity_vector_index \ - WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", - ) - .bind(tenant) - .bind(entity_type) - .bind(entity_id) - .execute(&mut *tx) - .await - .map_err(|e| PersistenceError::Storage(e.to_string()))?; - for row in vector_rows { - crate::dbm::postgres_query!( - "INSERT INTO entity_vector_index \ - (tenant, entity_type, decl_name, model_tag, entity_id, vector, sequence_nr) \ - VALUES ($1, $2, $3, $4, $5, $6, $7)", - ) - .bind(tenant) - .bind(entity_type) - .bind(&row.decl_name) - .bind(&row.model_tag) - .bind(entity_id) - .bind(pack_f32_le(&row.vector)) - .bind(new_seq as i64) - .execute(&mut *tx) - .await - .map_err(|e| PersistenceError::Storage(e.to_string()))?; - } - } - - let commit_started = Instant::now(); - tx.commit().await.map_err(|e| { - record_postgres_transaction_commit_duration( - commit_started.elapsed(), - EVENT_APPEND_OPERATION, - "error", - ); - PersistenceError::Storage(e.to_string()) - })?; - record_postgres_transaction_commit_duration( - commit_started.elapsed(), - EVENT_APPEND_OPERATION, - "ok", - ); - transaction_timer.set_outcome("ok"); - - Ok(new_seq) } async fn backfill_entity_keys( @@ -305,72 +232,20 @@ impl EventStore for PostgresEventStore { tenant: &str, entity_type: &str, entity_id: &str, + expected_sequence: u64, + contract_fence: temper_runtime::persistence::KeyIndexBackfillFence<'_>, key_rows: &[temper_runtime::persistence::EntityKeyRow], ) -> Result<(), PersistenceError> { - if key_rows.is_empty() { - return Ok(()); - } - let mut tx = self - .pool - .begin() - .await - .map_err(|e| PersistenceError::Storage(e.to_string()))?; - for key in key_rows { - // A different entity already holding this key is a pre-existing data - // conflict — log and skip (don't fail the whole backfill on one row; - // the conflict surfaces via the metric and a keyed read still resolves - // to whoever currently holds it). - let holder: Option<(String,)> = crate::dbm::postgres_query_as!( - "SELECT entity_id FROM entity_key_index \ - WHERE tenant = $1 AND entity_type = $2 AND key_name = $3 AND key_hash = $4", - ) - .bind(tenant) - .bind(entity_type) - .bind(&key.key_name) - .bind(&key.key_hash) - .fetch_optional(&mut *tx) - .await - .map_err(|e| PersistenceError::Storage(e.to_string()))?; - if let Some((existing,)) = &holder - && existing != entity_id - { - tracing::warn!( - tenant, entity_type, entity_id, existing, - key_name = %key.key_name, - "entity_key_index backfill: declared-key conflict; skipping" - ); - continue; - } - crate::dbm::postgres_query!( - "DELETE FROM entity_key_index \ - WHERE tenant = $1 AND entity_type = $2 AND key_name = $3 AND entity_id = $4", - ) - .bind(tenant) - .bind(entity_type) - .bind(&key.key_name) - .bind(entity_id) - .execute(&mut *tx) - .await - .map_err(|e| PersistenceError::Storage(e.to_string()))?; - crate::dbm::postgres_query!( - "INSERT INTO entity_key_index \ - (tenant, entity_type, key_name, key_hash, entity_id, sequence_nr) \ - VALUES ($1, $2, $3, $4, $5, 0) \ - ON CONFLICT (tenant, entity_type, key_name, key_hash) DO NOTHING", - ) - .bind(tenant) - .bind(entity_type) - .bind(&key.key_name) - .bind(&key.key_hash) - .bind(entity_id) - .execute(&mut *tx) - .await - .map_err(|e| PersistenceError::Storage(e.to_string()))?; - } - tx.commit() - .await - .map_err(|e| PersistenceError::Storage(e.to_string()))?; - Ok(()) + key_index::backfill_entity_keys( + &self.pool, + tenant, + entity_type, + entity_id, + expected_sequence, + contract_fence, + key_rows, + ) + .await } async fn mark_key_index_backfilled( @@ -379,67 +254,87 @@ impl EventStore for PostgresEventStore { entity_type: &str, key_set: &str, ) -> Result<(), PersistenceError> { - let mut conn = self - .pool - .acquire() - .await - .map_err(|e| PersistenceError::Storage(e.to_string()))?; - // Upsert the covered key-set: a re-key after a key-set change must OVERWRITE the - // stale set (not DO NOTHING) so `complete` reflects the keys actually assigned. - crate::dbm::postgres_query!( - "INSERT INTO key_index_backfill_watermark (tenant, entity_type, key_set) \ - VALUES ($1, $2, $3) \ - ON CONFLICT (tenant, entity_type) \ - DO UPDATE SET key_set = EXCLUDED.key_set, completed_at = now()", - ) - .bind(tenant) - .bind(entity_type) - .bind(key_set) - .execute(&mut *conn) - .await - .map_err(|e| PersistenceError::Storage(e.to_string()))?; - Ok(()) + key_index::mark_backfilled(&self.pool, tenant, entity_type, key_set).await } async fn key_index_backfilled_types( &self, tenant: &str, ) -> Result, PersistenceError> { - let mut conn = self - .pool - .acquire() - .await - .map_err(|e| PersistenceError::Storage(e.to_string()))?; - let rows: Vec<(String, String)> = crate::dbm::postgres_query_as!( - "SELECT entity_type, key_set FROM key_index_backfill_watermark WHERE tenant = $1", + key_index::backfilled_types(&self.pool, tenant).await + } + + async fn key_index_activated_contracts( + &self, + ) -> Result, PersistenceError> { + key_index::activated_contracts(&self.pool).await + } + + async fn key_index_reconciliation_revision( + &self, + tenant: &str, + entity_type: &str, + ) -> Result { + key_index::reconciliation_revision(&self.pool, tenant, entity_type).await + } + + async fn begin_key_index_backfill( + &self, + tenant: &str, + entity_type: &str, + key_set: &str, + ) -> Result { + key_index::begin_backfill(&self.pool, tenant, entity_type, key_set).await + } + + async fn activate_key_index_contract( + &self, + tenant: &str, + entity_type: &str, + key_set: &str, + purge_existing_rows: bool, + ) -> Result { + key_index::activate_contract( + &self.pool, + tenant, + entity_type, + key_set, + purge_existing_rows, ) - .bind(tenant) - .fetch_all(&mut *conn) .await - .map_err(|e| PersistenceError::Storage(e.to_string()))?; - Ok(rows.into_iter().collect()) } - async fn keyed_entity_ids_for_type( + async fn activate_key_index_contracts( + &self, + tenant: &str, + activations: &[KeyContractActivation], + ) -> Result, PersistenceError> { + key_index::activate_contracts(&self.pool, tenant, activations).await + } + + async fn mark_key_index_backfilled_if_revision( &self, tenant: &str, entity_type: &str, - ) -> Result, PersistenceError> { - let mut conn = self - .pool - .acquire() - .await - .map_err(|e| PersistenceError::Storage(e.to_string()))?; - let rows: Vec<(String,)> = crate::dbm::postgres_query_as!( - "SELECT DISTINCT entity_id FROM entity_key_index \ - WHERE tenant = $1 AND entity_type = $2", + key_set: &str, + expected_revision: u64, + ) -> Result { + key_index::mark_backfilled_if_revision( + &self.pool, + tenant, + entity_type, + key_set, + expected_revision, ) - .bind(tenant) - .bind(entity_type) - .fetch_all(&mut *conn) .await - .map_err(|e| PersistenceError::Storage(e.to_string()))?; - Ok(rows.into_iter().map(|(entity_id,)| entity_id).collect()) + } + + async fn keyed_entity_ids_for_type( + &self, + tenant: &str, + entity_type: &str, + ) -> Result, PersistenceError> { + key_index::keyed_entity_ids(&self.pool, tenant, entity_type).await } async fn lookup_by_key( @@ -449,23 +344,20 @@ impl EventStore for PostgresEventStore { key_name: &str, key_hash: &str, ) -> Result, PersistenceError> { - let mut conn = self - .pool - .acquire() - .await - .map_err(|e| PersistenceError::Storage(e.to_string()))?; - let row: Option<(String,)> = crate::dbm::postgres_query_as!( - "SELECT entity_id FROM entity_key_index \ - WHERE tenant = $1 AND entity_type = $2 AND key_name = $3 AND key_hash = $4", - ) - .bind(tenant) - .bind(entity_type) - .bind(key_name) - .bind(key_hash) - .fetch_optional(&mut *conn) - .await - .map_err(|e| PersistenceError::Storage(e.to_string()))?; - Ok(row.map(|(id,)| id)) + Ok(self + .lookup_by_key_with_sequence(tenant, entity_type, key_name, key_hash) + .await? + .map(|lookup| lookup.entity_id)) + } + + async fn lookup_by_key_with_sequence( + &self, + tenant: &str, + entity_type: &str, + key_name: &str, + key_hash: &str, + ) -> Result, PersistenceError> { + key_index::lookup(&self.pool, tenant, entity_type, key_name, key_hash).await } async fn backfill_entity_vectors( @@ -616,178 +508,12 @@ impl EventStore for PostgresEventStore { Ok(rows.into_iter().map(|(entity_id,)| entity_id).collect()) } - /// Atomically append to multiple entity journals in one PostgreSQL - /// transaction. Used as the storage foundation for cross-actor Composite - /// transactions: every stream's optimistic-concurrency check must pass - /// before any event is inserted. + /// Atomically append to multiple entity journals in one PostgreSQL transaction. async fn append_batch( &self, appends: &[PersistenceAppend], ) -> Result, PersistenceError> { - if appends.is_empty() { - return Ok(Vec::new()); - } - - let mut seen = std::collections::BTreeSet::new(); - for append in appends { - if !seen.insert(append.persistence_id.as_str()) { - return Err(PersistenceError::Storage(format!( - "duplicate persistence_id '{}' in append_batch", - append.persistence_id - ))); - } - } - - let mut transaction_timer = PostgresTransactionTimer::start(EVENT_APPEND_OPERATION); - let acquire_started = Instant::now(); - let mut conn = match self.pool.acquire().await { - Ok(conn) => { - record_postgres_pool_acquire_duration( - acquire_started.elapsed(), - EVENT_APPEND_OPERATION, - "ok", - ); - conn - } - Err(e) => { - record_postgres_pool_acquire_duration( - acquire_started.elapsed(), - EVENT_APPEND_OPERATION, - "error", - ); - return Err(PersistenceError::Storage(e.to_string())); - } - }; - let begin_started = Instant::now(); - let mut tx = match conn.begin().await { - Ok(tx) => { - record_postgres_transaction_begin_duration( - begin_started.elapsed(), - EVENT_APPEND_OPERATION, - "ok", - ); - tx - } - Err(e) => { - record_postgres_transaction_begin_duration( - begin_started.elapsed(), - EVENT_APPEND_OPERATION, - "error", - ); - return Err(PersistenceError::Storage(e.to_string())); - } - }; - - let mut parsed = Vec::with_capacity(appends.len()); - for append in appends { - let (tenant, entity_type, entity_id) = - parse_persistence_id_parts(&append.persistence_id) - .map_err(PersistenceError::Storage)?; - let row: Option<(i64,)> = crate::dbm::postgres_query_as!( - "SELECT COALESCE(MAX(sequence_nr), 0) FROM events \ - WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", - ) - .bind(tenant) - .bind(entity_type) - .bind(entity_id) - .fetch_optional(&mut *tx) - .await - .map_err(|e| PersistenceError::Storage(e.to_string()))?; - - let current_seq = row.map(|r| r.0 as u64).unwrap_or(0); - if current_seq != append.expected_sequence { - transaction_timer.set_outcome("concurrency_violation"); - return Err(PersistenceError::ConcurrencyViolation { - expected: append.expected_sequence, - actual: current_seq, - }); - } - let segment_index = segments::open_segment_for_append( - &mut tx, - tenant, - entity_type, - entity_id, - current_seq, - ) - .await?; - parsed.push(( - tenant.to_string(), - entity_type.to_string(), - entity_id.to_string(), - segment_index, - )); - } - - let mut results = Vec::with_capacity(appends.len()); - for (append, (tenant, entity_type, entity_id, segment_index)) in - appends.iter().zip(parsed.iter()) - { - let mut new_seq = append.expected_sequence; - for event in &append.events { - new_seq += 1; - let metadata_json = serde_json::to_value(&event.metadata) - .map_err(|e| PersistenceError::Serialization(e.to_string()))?; - - if let Err(e) = crate::dbm::postgres_query!( - "INSERT INTO events (tenant, entity_type, entity_id, sequence_nr, segment_index, event_type, payload, metadata) \ - VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", - ) - .bind(tenant) - .bind(entity_type) - .bind(entity_id) - .bind(new_seq as i64) - .bind(*segment_index) - .bind(&event.event_type) - .bind(&event.payload) - .bind(metadata_json) - .execute(&mut *tx) - .await - { - let msg = e.to_string(); - if msg.contains("unique") || msg.contains("duplicate key") { - transaction_timer.set_outcome("concurrency_violation"); - return Err(PersistenceError::ConcurrencyViolation { - expected: append.expected_sequence, - actual: new_seq, - }); - } - return Err(PersistenceError::Storage(msg)); - } - } - if new_seq > append.expected_sequence { - segments::update_segment_after_append( - &mut tx, - tenant, - entity_type, - entity_id, - *segment_index, - new_seq, - ) - .await?; - } - results.push(PersistenceAppendResult { - persistence_id: append.persistence_id.clone(), - sequence_nr: new_seq, - }); - } - - let commit_started = Instant::now(); - tx.commit().await.map_err(|e| { - record_postgres_transaction_commit_duration( - commit_started.elapsed(), - EVENT_APPEND_OPERATION, - "error", - ); - PersistenceError::Storage(e.to_string()) - })?; - record_postgres_transaction_commit_duration( - commit_started.elapsed(), - EVENT_APPEND_OPERATION, - "ok", - ); - transaction_timer.set_outcome("ok"); - - Ok(results) + self.append_batch_inner(appends).await } /// Read events from the journal starting after `from_sequence`. @@ -830,63 +556,121 @@ impl EventStore for PostgresEventStore { .collect() } - /// Save (upsert) a snapshot for the given entity. - /// - /// Uses `ON CONFLICT … DO UPDATE` so that only the latest snapshot is - /// retained per entity. - async fn save_snapshot( + async fn read_events_page( &self, persistence_id: &str, - sequence_nr: u64, - snapshot: &[u8], - ) -> Result<(), PersistenceError> { + from_sequence: u64, + through_sequence: u64, + limit: usize, + ) -> Result, PersistenceError> { + assert!(limit > 0, "event page limit must be positive"); + assert!( + through_sequence >= from_sequence, + "event page boundary must not precede its cursor" + ); let (tenant, entity_type, entity_id) = parse_persistence_id_parts(persistence_id).map_err(PersistenceError::Storage)?; + let from_sequence = i64::try_from(from_sequence).map_err(|_| { + PersistenceError::Storage("event page cursor exceeds PostgreSQL bigint".to_string()) + })?; + let through_sequence = i64::try_from(through_sequence).map_err(|_| { + PersistenceError::Storage("event page boundary exceeds PostgreSQL bigint".to_string()) + })?; + let limit = i64::try_from(limit).map_err(|_| { + PersistenceError::Storage("event page limit exceeds PostgreSQL bigint".to_string()) + })?; - let mut tx = self - .pool - .begin() + let rows: Vec<(i64, String, serde_json::Value, serde_json::Value)> = + crate::dbm::postgres_query_as!( + "SELECT sequence_nr, event_type, payload, metadata \ + FROM events \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ + AND sequence_nr > $4 AND sequence_nr <= $5 \ + ORDER BY sequence_nr ASC \ + LIMIT $6", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .bind(from_sequence) + .bind(through_sequence) + .bind(limit) + .fetch_all(&self.pool) .await - .map_err(|e| PersistenceError::Storage(e.to_string()))?; + .map_err(|error| PersistenceError::Storage(error.to_string()))?; - crate::dbm::postgres_query!( - "INSERT INTO snapshots (tenant, entity_type, entity_id, sequence_nr, state) \ - VALUES ($1, $2, $3, $4, $5) \ - ON CONFLICT (tenant, entity_type, entity_id) \ - DO UPDATE SET sequence_nr = $4, state = $5, created_at = now()", + rows.into_iter() + .map(|(sequence_nr, event_type, payload, metadata)| { + let metadata = serde_json::from_value(metadata) + .map_err(|error| PersistenceError::Serialization(error.to_string()))?; + Ok(PersistenceEnvelope { + sequence_nr: sequence_nr as u64, + event_type, + payload, + metadata, + }) + }) + .collect() + } + + async fn journal_boundary( + &self, + persistence_id: &str, + ) -> Result { + let (tenant, entity_type, entity_id) = + parse_persistence_id_parts(persistence_id).map_err(PersistenceError::Storage)?; + let row: (i64, Option) = crate::dbm::postgres_query_as!( + "SELECT COALESCE(MAX(sequence_nr), 0), \ + MIN(sequence_nr) FILTER (WHERE \ + CASE WHEN jsonb_typeof(payload) = 'object' AND payload ? 'to_status' \ + THEN payload ->> 'to_status' = 'Deleted' \ + ELSE event_type = 'Deleted' OR payload ->> 'action' = 'Deleted' \ + END) \ + FROM events \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", ) .bind(tenant) .bind(entity_type) .bind(entity_id) - .bind(sequence_nr as i64) - .bind(snapshot) - .execute(&mut *tx) + .fetch_one(&self.pool) .await .map_err(|e| PersistenceError::Storage(e.to_string()))?; + Ok(JournalBoundary { + latest_sequence: row.0 as u64, + first_terminal_sequence: row.1.map(|sequence_nr| sequence_nr as u64), + }) + } - crate::dbm::postgres_query!( - "INSERT INTO snapshot_history (tenant, entity_type, entity_id, sequence_nr, state) \ - VALUES ($1, $2, $3, $4, $5) \ - ON CONFLICT (tenant, entity_type, entity_id, sequence_nr) \ - DO UPDATE SET state = $5, created_at = now()", + /// Save (upsert) a snapshot for the given entity. + /// + /// Uses `ON CONFLICT … DO UPDATE` so that only the latest snapshot is + /// retained per entity. + async fn save_snapshot( + &self, + persistence_id: &str, + sequence_nr: u64, + snapshot: &[u8], + ) -> Result<(), PersistenceError> { + self.save_snapshot_inner( + persistence_id, + sequence_nr, + snapshot, + &SnapshotSourceFence::Unchecked, + None, ) - .bind(tenant) - .bind(entity_type) - .bind(entity_id) - .bind(sequence_nr as i64) - .bind(snapshot) - .execute(&mut *tx) .await - .map_err(|e| PersistenceError::Storage(e.to_string()))?; - - segments::rotate_after_snapshot(&mut tx, tenant, entity_type, entity_id, sequence_nr) - .await?; + } - tx.commit() + async fn save_snapshot_if_source( + &self, + persistence_id: &str, + sequence_nr: u64, + snapshot: &[u8], + source: &SnapshotSourceFence, + key_contract: Option<&str>, + ) -> Result<(), PersistenceError> { + self.save_snapshot_inner(persistence_id, sequence_nr, snapshot, source, key_contract) .await - .map_err(|e| PersistenceError::Storage(e.to_string()))?; - - Ok(()) } /// Load the latest snapshot for an entity. @@ -951,7 +735,10 @@ impl EventStore for PostgresEventStore { WHERE d.tenant = c.tenant \ AND d.entity_type = c.entity_type \ AND d.entity_id = c.entity_id \ - AND d.event_type = 'Deleted' \ + AND (CASE WHEN jsonb_typeof(d.payload) = 'object' AND d.payload ? 'to_status' \ + THEN d.payload ->> 'to_status' = 'Deleted' \ + ELSE d.event_type = 'Deleted' OR d.payload ->> 'action' = 'Deleted' \ + END) \ ) \ UNION \ SELECT f.entity_id \ @@ -964,7 +751,26 @@ impl EventStore for PostgresEventStore { WHERE d.tenant = f.tenant \ AND d.entity_type = f.entity_type \ AND d.entity_id = f.entity_id \ - AND d.event_type = 'Deleted' \ + AND (CASE WHEN jsonb_typeof(d.payload) = 'object' AND d.payload ? 'to_status' \ + THEN d.payload ->> 'to_status' = 'Deleted' \ + ELSE d.event_type = 'Deleted' OR d.payload ->> 'action' = 'Deleted' \ + END) \ + ) \ + UNION \ + SELECT s.entity_id \ + FROM snapshots s \ + WHERE s.tenant = $1 \ + AND s.entity_type = $2 \ + AND NOT EXISTS ( \ + SELECT 1 \ + FROM events d \ + WHERE d.tenant = s.tenant \ + AND d.entity_type = s.entity_type \ + AND d.entity_id = s.entity_id \ + AND (CASE WHEN jsonb_typeof(d.payload) = 'object' AND d.payload ? 'to_status' \ + THEN d.payload ->> 'to_status' = 'Deleted' \ + ELSE d.event_type = 'Deleted' OR d.payload ->> 'action' = 'Deleted' \ + END) \ ) \ UNION \ SELECT DISTINCT e.entity_id \ @@ -977,7 +783,10 @@ impl EventStore for PostgresEventStore { WHERE d.tenant = e.tenant \ AND d.entity_type = e.entity_type \ AND d.entity_id = e.entity_id \ - AND d.event_type = 'Deleted' \ + AND (CASE WHEN jsonb_typeof(d.payload) = 'object' AND d.payload ? 'to_status' \ + THEN d.payload ->> 'to_status' = 'Deleted' \ + ELSE d.event_type = 'Deleted' OR d.payload ->> 'action' = 'Deleted' \ + END) \ ) \ ) ids \ ORDER BY entity_id", @@ -991,6 +800,147 @@ impl EventStore for PostgresEventStore { Ok(rows) } + async fn list_entity_ids_for_key_reconciliation( + &self, + tenant: &str, + entity_type: &str, + ) -> Result, PersistenceError> { + // Repair authority is deliberately broader than live query enumeration: + // deleted streams can retain pre-v3 rows, while interrupted/manual legacy + // writes can leave key rows whose journal no longer exists. Migration-era + // catalog, field-index, and snapshot rows may also be the only durable copy + // of an entity. Every source must therefore participate in repair coverage. + let rows: Vec = crate::dbm::postgres_query_scalar!( + "SELECT entity_id \ + FROM ( \ + SELECT DISTINCT e.entity_id \ + FROM events e \ + WHERE e.tenant = $1 AND e.entity_type = $2 \ + UNION \ + SELECT DISTINCT k.entity_id \ + FROM entity_key_index k \ + WHERE k.tenant = $1 AND k.entity_type = $2 \ + UNION \ + SELECT c.entity_id \ + FROM entity_catalog c \ + WHERE c.tenant = $1 AND c.entity_type = $2 \ + UNION \ + SELECT DISTINCT f.entity_id \ + FROM entity_field_index f \ + WHERE f.tenant = $1 AND f.entity_type = $2 \ + UNION \ + SELECT s.entity_id \ + FROM snapshots s \ + WHERE s.tenant = $1 AND s.entity_type = $2 \ + ) repair_ids \ + ORDER BY entity_id", + ) + .bind(tenant) + .bind(entity_type) + .fetch_all(&self.pool) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + + Ok(rows) + } + + async fn list_key_reconciliation_page( + &self, + tenant: &str, + entity_type: &str, + after_entity_id: Option<&str>, + through_entity_id: &str, + limit: usize, + ) -> Result, PersistenceError> { + assert!(limit > 0, "key reconciliation page limit must be positive"); + let limit = limit.min(i64::MAX as usize) as i64; + let rows: Vec<(String, bool)> = crate::dbm::postgres_query_as!( + "WITH repair_ids AS ( \ + SELECT DISTINCT e.entity_id FROM events e \ + WHERE e.tenant = $1 AND e.entity_type = $2 \ + UNION \ + SELECT DISTINCT k.entity_id FROM entity_key_index k \ + WHERE k.tenant = $1 AND k.entity_type = $2 \ + UNION \ + SELECT c.entity_id FROM entity_catalog c \ + WHERE c.tenant = $1 AND c.entity_type = $2 \ + UNION \ + SELECT DISTINCT f.entity_id FROM entity_field_index f \ + WHERE f.tenant = $1 AND f.entity_type = $2 \ + UNION \ + SELECT s.entity_id FROM snapshots s \ + WHERE s.tenant = $1 AND s.entity_type = $2 \ + ), source_ids AS ( \ + SELECT DISTINCT e.entity_id FROM events e \ + WHERE e.tenant = $1 AND e.entity_type = $2 \ + UNION \ + SELECT c.entity_id FROM entity_catalog c \ + WHERE c.tenant = $1 AND c.entity_type = $2 \ + UNION \ + SELECT DISTINCT f.entity_id FROM entity_field_index f \ + WHERE f.tenant = $1 AND f.entity_type = $2 \ + UNION \ + SELECT s.entity_id FROM snapshots s \ + WHERE s.tenant = $1 AND s.entity_type = $2 \ + ) \ + SELECT r.entity_id, \ + EXISTS (SELECT 1 FROM source_ids s WHERE s.entity_id = r.entity_id) \ + AND NOT EXISTS ( \ + SELECT 1 FROM events d \ + WHERE d.tenant = $1 AND d.entity_type = $2 \ + AND d.entity_id = r.entity_id \ + AND (CASE WHEN jsonb_typeof(d.payload) = 'object' AND d.payload ? 'to_status' \ + THEN d.payload ->> 'to_status' = 'Deleted' \ + ELSE d.event_type = 'Deleted' OR d.payload ->> 'action' = 'Deleted' \ + END) \ + ) AS is_live \ + FROM repair_ids r \ + WHERE ($3::text IS NULL OR r.entity_id > $3) \ + AND r.entity_id <= $4 \ + ORDER BY r.entity_id \ + LIMIT $5", + ) + .bind(tenant) + .bind(entity_type) + .bind(after_entity_id) + .bind(through_entity_id) + .bind(limit) + .fetch_all(&self.pool) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + + Ok(rows + .into_iter() + .map( + |(entity_id, is_live)| temper_runtime::persistence::KeyReconciliationEntity { + entity_id, + is_live, + }, + ) + .collect()) + } + + async fn key_reconciliation_boundary( + &self, + tenant: &str, + entity_type: &str, + ) -> Result, PersistenceError> { + crate::dbm::postgres_query_scalar!( + "SELECT MAX(entity_id) FROM ( \ + SELECT DISTINCT e.entity_id FROM events e WHERE e.tenant = $1 AND e.entity_type = $2 \ + UNION SELECT DISTINCT k.entity_id FROM entity_key_index k WHERE k.tenant = $1 AND k.entity_type = $2 \ + UNION SELECT c.entity_id FROM entity_catalog c WHERE c.tenant = $1 AND c.entity_type = $2 \ + UNION SELECT DISTINCT f.entity_id FROM entity_field_index f WHERE f.tenant = $1 AND f.entity_type = $2 \ + UNION SELECT s.entity_id FROM snapshots s WHERE s.tenant = $1 AND s.entity_type = $2 \ + ) repair_ids", + ) + .bind(tenant) + .bind(entity_type) + .fetch_one(&self.pool) + .await + .map_err(|error| PersistenceError::Storage(error.to_string())) + } + async fn list_entity_ids_limited( &self, tenant: &str, @@ -1042,442 +992,3 @@ impl EventStore for PostgresEventStore { #[cfg(test)] #[path = "store_projection_test.rs"] mod projection_tests; - -#[cfg(test)] -mod tests { - use super::*; - use crate::migration::run_migrations; - use temper_runtime::tenant::parse_persistence_id_parts; - - fn parse_pid(persistence_id: &str) -> Result<(&str, &str, &str), PersistenceError> { - parse_persistence_id_parts(persistence_id).map_err(PersistenceError::Storage) - } - - fn test_envelope(event_type: &str, payload: serde_json::Value) -> PersistenceEnvelope { - PersistenceEnvelope { - sequence_nr: 0, - event_type: event_type.to_string(), - payload, - metadata: EventMetadata { - event_id: uuid::Uuid::new_v4(), - causation_id: uuid::Uuid::new_v4(), - correlation_id: uuid::Uuid::new_v4(), - timestamp: chrono::Utc::now(), - actor_id: "store-test".to_string(), - }, - } - } - - // -- persistence_id parsing --------------------------------------------- - - #[test] - fn parse_3_segment_persistence_id() { - let (tenant, entity_type, entity_id) = - parse_pid("alpha:Order:abc-123").expect("valid three-segment persistence ID"); - assert_eq!(tenant, "alpha"); - assert_eq!(entity_type, "Order"); - assert_eq!(entity_id, "abc-123"); - } - - #[test] - fn parse_legacy_2_segment_persistence_id() { - let (tenant, entity_type, entity_id) = - parse_pid("Order:abc-123").expect("valid legacy two-segment persistence ID"); - assert_eq!(tenant, "default"); - assert_eq!(entity_type, "Order"); - assert_eq!(entity_id, "abc-123"); - } - - #[test] - fn parse_3_segment_with_colons_in_id() { - // splitn(3, ':') puts everything after the second colon into entity_id - let (tenant, entity_type, entity_id) = - parse_pid("beta:Task:T-1:sub").expect("valid persistence ID with colon in entity ID"); - assert_eq!(tenant, "beta"); - assert_eq!(entity_type, "Task"); - assert_eq!(entity_id, "T-1:sub"); - } - - #[test] - fn parse_persistence_id_missing_colon() { - let err = parse_pid("OrderAbc123").unwrap_err(); - assert!( - matches!(err, PersistenceError::Storage(_)), - "expected Storage error, got: {err:?}" - ); - } - - #[test] - fn parse_persistence_id_empty_segment() { - assert!(parse_pid(":Order:abc").is_err()); - assert!(parse_pid("tenant::abc").is_err()); - assert!(parse_pid("tenant:Order:").is_err()); - assert!(parse_pid(":abc").is_err()); - assert!(parse_pid("Order:").is_err()); - } - - #[test] - fn list_entity_ids_returns_distinct_pairs() { - let database_url = match std::env::var("DATABASE_URL") { - Ok(url) => url, - Err(_) => { - eprintln!("skipping Postgres integration test: DATABASE_URL is not set"); - return; - } - }; - - sqlx::test_block_on(async { - let pool = PgPool::connect(&database_url) - .await - .expect("connect to DATABASE_URL"); - run_migrations(&pool).await.expect("run migrations"); - let store = PostgresEventStore::new(pool); - - let tenant_a = format!("tenant-a-{}", uuid::Uuid::new_v4()); - let tenant_b = format!("tenant-b-{}", uuid::Uuid::new_v4()); - - let order_1 = format!("{tenant_a}:Order:ord-1"); - let order_2 = format!("{tenant_a}:Order:ord-2"); - let task_1 = format!("{tenant_a}:Task:task-1"); - let other_tenant = format!("{tenant_b}:Order:ord-9"); - - store - .append( - &order_1, - 0, - &[test_envelope( - "OrderCreated", - serde_json::json!({"id":"ord-1"}), - )], - ) - .await - .expect("append first order event"); - store - .append( - &order_1, - 1, - &[test_envelope( - "OrderUpdated", - serde_json::json!({"step": 2}), - )], - ) - .await - .expect("append order update event"); - store - .append( - &order_2, - 0, - &[test_envelope( - "OrderCreated", - serde_json::json!({"id":"ord-2"}), - )], - ) - .await - .expect("append second order event"); - store - .append( - &task_1, - 0, - &[test_envelope( - "TaskCreated", - serde_json::json!({"id":"task-1"}), - )], - ) - .await - .expect("append task event"); - store - .append( - &other_tenant, - 0, - &[test_envelope( - "OrderCreated", - serde_json::json!({"id":"ord-9"}), - )], - ) - .await - .expect("append other tenant event"); - - let mut entities = store - .list_entity_ids(&tenant_a) - .await - .expect("list tenant entity IDs"); - entities.sort(); - - assert_eq!( - entities, - vec![ - ("Order".to_string(), "ord-1".to_string()), - ("Order".to_string(), "ord-2".to_string()), - ("Task".to_string(), "task-1".to_string()), - ] - ); - }); - } - - #[test] - fn postgres_platform_methods_are_part_of_the_store_surface() { - // Compile-only check: the function body is never executed, so the - // unawaited futures are intentional. Clippy's let_underscore_future - // lint catches forgotten awaits in real code; here it's a false - // positive. - #[allow(clippy::let_underscore_future)] - fn assert_methods(store: &PostgresEventStore) { - let _ = store.upsert_query_projection( - "tenant", - "Session", - "s-1", - "Running", - &serde_json::json!({"phase":"Running"}), - 1, - ); - let _ = store.remove_query_projection("tenant", "Session", "s-1"); - let _ = store.query_field_index( - "tenant", - "Session", - "fields @> $3::jsonb", - vec!["{\"phase\":\"Running\"}".to_string()], - ); - let _ = store.persist_trajectory(crate::PostgresTrajectoryInsert { - tenant: "tenant", - entity_type: "Session", - entity_id: "s-1", - action: "ProgressMade", - success: true, - from_status: Some("Running"), - to_status: Some("Running"), - error: None, - agent_id: Some("agent"), - session_id: Some("session"), - authz_denied: None, - denied_resource: None, - denied_module: None, - source: Some("Entity"), - spec_governed: Some(true), - created_at: "2026-04-28T00:00:00Z", - request_body: Some("{\"ok\":true}"), - intent: Some("test"), - matched_policy_ids: Some("[\"policy:test\"]"), - }); - let _ = store.save_policy( - "tenant", - "primary", - "permit(principal, action, resource);", - "test", - ); - let _ = store.load_policies_for_tenant("tenant"); - let _ = store.load_all_policies(); - let _ = store.toggle_policy_enabled("tenant", "primary", true); - let _ = store.update_policy_text( - "tenant", - "primary", - "permit(principal, action, resource);", - "test", - ); - let _ = store.delete_policy("tenant", "primary"); - } - - let _ = assert_methods; - } - - #[test] - fn postgres_long_tail_methods_are_part_of_the_store_surface() { - let _ = PostgresEventStore::load_recent_trajectories; - let _ = PostgresEventStore::load_unmet_intent_rows; - let _ = PostgresEventStore::load_submit_spec_timestamps; - let _ = PostgresEventStore::count_trajectories_by_tenant; - let _ = PostgresEventStore::query_trajectory_stats; - let _ = PostgresEventStore::query_trajectories_by_agent; - let _ = PostgresEventStore::query_agent_summaries; - - let _ = PostgresEventStore::upsert_feature_request; - let _ = PostgresEventStore::list_feature_requests; - let _ = PostgresEventStore::update_feature_request; - let _ = PostgresEventStore::insert_evolution_record; - let _ = PostgresEventStore::get_evolution_record; - let _ = PostgresEventStore::list_evolution_records; - let _ = PostgresEventStore::list_ranked_insights; - let _ = PostgresEventStore::insert_design_time_event; - let _ = PostgresEventStore::list_design_time_events; - - let _ = PostgresEventStore::persist_ots_trajectory; - let _ = PostgresEventStore::list_ots_trajectories; - let _ = PostgresEventStore::get_ots_trajectory; - - let _ = PostgresEventStore::put_blob; - let _ = PostgresEventStore::put_blob_with_ttl; - let _ = PostgresEventStore::get_blob; - let _ = PostgresEventStore::sweep_expired_blobs; - - let _ = PostgresEventStore::upsert_secret; - let _ = PostgresEventStore::delete_secret; - let _ = PostgresEventStore::load_secrets_for_tenant; - - let _ = PostgresEventStore::upsert_policy_denial_pattern; - let _ = PostgresEventStore::load_policy_denial_patterns; - - let _ = PostgresEventStore::query_decisions; - let _ = PostgresEventStore::query_all_decisions; - let _ = PostgresEventStore::get_pending_decision; - - let _ = PostgresEventStore::load_wasm_module; - let _ = PostgresEventStore::load_wasm_module_metadata_all_tenants; - let _ = PostgresEventStore::persist_wasm_invocation; - let _ = PostgresEventStore::load_recent_wasm_invocations; - let _ = PostgresEventStore::delete_wasm_module; - - let _ = PostgresEventStore::upsert_published_artifact; - let _ = PostgresEventStore::load_published_artifact; - } - - #[test] - fn postgres_query_projection_batch_method_is_part_of_the_store_surface() { - let _ = PostgresEventStore::load_query_projection_fields_many; - let _ = PostgresEventStore::load_selected_entity_catalog_rows_pg; - } - - #[test] - fn load_query_projection_fields_many_returns_requested_fields() { - let database_url = match std::env::var("DATABASE_URL") { - Ok(url) => url, - Err(_) => { - eprintln!("skipping Postgres integration test: DATABASE_URL is not set"); - return; - } - }; - - sqlx::test_block_on(async { - let pool = PgPool::connect(&database_url) - .await - .expect("connect to DATABASE_URL"); - run_migrations(&pool).await.expect("run migrations"); - let store = PostgresEventStore::new(pool.clone()); - let tenant = format!("tenant-projection-{}", uuid::Uuid::new_v4()); - - store - .upsert_query_projection( - &tenant, - "File", - "file-a", - "Ready", - &serde_json::json!({ - "content_hash": "sha256:file-a", - "mime_type": "text/plain", - "has_content": true, - }), - 12, - ) - .await - .expect("upsert query projection row"); - - let rows = store - .load_query_projection_fields_many( - &tenant, - "File", - &["file-a".to_string(), "missing".to_string()], - &["content_hash", "has_content"], - ) - .await - .expect("load requested projection fields"); - - assert_eq!(rows.len(), 1); - assert_eq!(rows[0].entity_id, "file-a"); - assert_eq!(rows[0].status, "Ready"); - assert_eq!( - rows[0] - .fields - .get("content_hash") - .and_then(|value| value.as_deref()), - Some("sha256:file-a") - ); - assert_eq!( - rows[0] - .fields - .get("has_content") - .and_then(|value| value.as_deref()), - Some("true") - ); - - crate::dbm::postgres_query!("DELETE FROM entity_field_index WHERE tenant = $1") - .bind(&tenant) - .execute(&pool) - .await - .expect("delete test entity field index rows"); - crate::dbm::postgres_query!("DELETE FROM entity_catalog WHERE tenant = $1") - .bind(&tenant) - .execute(&pool) - .await - .expect("delete test entity catalog rows"); - }); - } - - #[test] - fn published_artifact_upsert_round_trips_and_updates_by_id() { - let database_url = match std::env::var("DATABASE_URL") { - Ok(url) => url, - Err(_) => return, - }; - - sqlx::test_block_on(async { - let pool = PgPool::connect(&database_url) - .await - .expect("connect to DATABASE_URL"); - run_migrations(&pool).await.expect("run migrations"); - let store = PostgresEventStore::new(pool.clone()); - let tenant = format!("tenant-published-artifact-{}", uuid::Uuid::new_v4()); - - let artifact = crate::PostgresPublishedArtifactUpsert { - id: "part-test", - tenant: &tenant, - source_file_id: "file-a", - source_file_version_id: "", - content_hash: "sha256:abc", - label: "latest", - mime_type: "text/markdown", - byte_length: 42, - public_storage_key: "published/file-a.md", - public_url: "https://assets.example/published/file-a.md", - owner_ref_type: "Doc", - owner_ref_id: "doc-a", - status: "published", - }; - - let row = store - .upsert_published_artifact(&artifact) - .await - .expect("upsert published artifact"); - assert_eq!(row.tenant, tenant); - assert_eq!(row.id, "part-test"); - assert_eq!(row.public_url, "https://assets.example/published/file-a.md"); - - let updated = crate::PostgresPublishedArtifactUpsert { - public_url: "https://assets.example/published/file-a-v2.md", - public_storage_key: "published/file-a-v2.md", - byte_length: 43, - ..artifact - }; - store - .upsert_published_artifact(&updated) - .await - .expect("update published artifact"); - - let loaded = store - .load_published_artifact(&tenant, "part-test") - .await - .expect("load published artifact") - .expect("artifact should load after upsert"); - - assert_eq!( - loaded.public_url, - "https://assets.example/published/file-a-v2.md" - ); - assert_eq!(loaded.public_storage_key, "published/file-a-v2.md"); - assert_eq!(loaded.byte_length, 43); - - crate::dbm::postgres_query!("DELETE FROM published_artifacts WHERE tenant = $1") - .bind(&tenant) - .execute(&pool) - .await - .expect("delete test published artifact rows"); - }); - } -} diff --git a/crates/temper-store-postgres/src/store/append.rs b/crates/temper-store-postgres/src/store/append.rs new file mode 100644 index 000000000..b9479505f --- /dev/null +++ b/crates/temper-store-postgres/src/store/append.rs @@ -0,0 +1,293 @@ +//! Source-fenced PostgreSQL journal append persistence. + +use super::*; + +impl PostgresEventStore { + pub(super) async fn append_with_index_rows_inner( + &self, + persistence_id: &str, + expected_sequence: u64, + events: &[PersistenceEnvelope], + key_rows: &[temper_runtime::persistence::EntityKeyRow], + vector_rows: &[EntityVectorRow], + reconciliation: IndexReconciliation, + ) -> Result { + let (tenant, entity_type, entity_id) = + parse_persistence_id_parts(persistence_id).map_err(PersistenceError::Storage)?; + + let mut transaction_timer = PostgresTransactionTimer::start(EVENT_APPEND_OPERATION); + let acquire_started = Instant::now(); + let mut conn = match self.pool.acquire().await { + Ok(conn) => { + record_postgres_pool_acquire_duration( + acquire_started.elapsed(), + EVENT_APPEND_OPERATION, + "ok", + ); + conn + } + Err(e) => { + record_postgres_pool_acquire_duration( + acquire_started.elapsed(), + EVENT_APPEND_OPERATION, + "error", + ); + return Err(PersistenceError::Storage(e.to_string())); + } + }; + let begin_started = Instant::now(); + let mut tx = match conn.begin().await { + Ok(tx) => { + record_postgres_transaction_begin_duration( + begin_started.elapsed(), + EVENT_APPEND_OPERATION, + "ok", + ); + tx + } + Err(e) => { + record_postgres_transaction_begin_duration( + begin_started.elapsed(), + EVENT_APPEND_OPERATION, + "error", + ); + return Err(PersistenceError::Storage(e.to_string())); + } + }; + + lock_key_contract(&mut tx, tenant, entity_type).await?; + let lock_key = event_stream_lock_key(tenant, entity_type, entity_id); + lock_event_stream(&mut tx, &lock_key).await?; + + let row: Option<(i64,)> = crate::dbm::postgres_query_as!( + "SELECT COALESCE(MAX(sequence_nr), 0) FROM events \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_optional(&mut *tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + + let current_seq = row.map(|r| r.0 as u64).unwrap_or(0); + if current_seq != expected_sequence { + transaction_timer.set_outcome("concurrency_violation"); + return Err(PersistenceError::ConcurrencyViolation { + expected: expected_sequence, + actual: current_seq, + }); + } + if !matches!( + &reconciliation.snapshot_source, + SnapshotSourceFence::Unchecked + ) { + let current_snapshot = + load_snapshot_for_update(&mut tx, tenant, entity_type, entity_id).await?; + if !snapshot_source_matches(current_snapshot.as_ref(), &reconciliation.snapshot_source) + { + return Err(PersistenceError::SnapshotGenerationChanged); + } + } + + // ADR-0153: validate declared-key uniqueness BEFORE writing the journal so + // a reject is atomic (the journal does not advance). A different entity + // already holding the key is the violation (reject + surface). + if reconciliation.keys { + for key in key_rows { + let holder: Option<(String,)> = crate::dbm::postgres_query_as!( + "SELECT entity_id FROM entity_key_index \ + WHERE tenant = $1 AND entity_type = $2 AND key_name = $3 AND key_hash = $4", + ) + .bind(tenant) + .bind(entity_type) + .bind(&key.key_name) + .bind(&key.key_hash) + .fetch_optional(&mut *tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + if let Some((existing,)) = holder + && existing != entity_id + { + return Err(PersistenceError::Storage(format!( + "duplicate declared key '{}' for {entity_type}: held by {existing}", + key.key_name + ))); + } + } + } + + if reconciliation.keys { + reconcile_key_contract_state( + &mut tx, + tenant, + entity_type, + reconciliation.key_set_signature.as_deref(), + None, + KeyContractUse::LiveWrite, + ) + .await?; + } else if !events.is_empty() { + invalidate_key_coverage_for_unreconciled_append(&mut tx, tenant, entity_type).await?; + } + + let segment_index = + segments::open_segment_for_append(&mut tx, tenant, entity_type, entity_id, current_seq) + .await?; + + let mut new_seq = expected_sequence; + for event in events { + new_seq += 1; + let metadata_json = serde_json::to_value(&event.metadata) + .map_err(|e| PersistenceError::Serialization(e.to_string()))?; + + if let Err(e) = crate::dbm::postgres_query!( + "INSERT INTO events (tenant, entity_type, entity_id, sequence_nr, segment_index, event_type, payload, metadata) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .bind(new_seq as i64) + .bind(segment_index) + .bind(&event.event_type) + .bind(&event.payload) + .bind(metadata_json) + .execute(&mut *tx) + .await + { + let msg = e.to_string(); + if msg.contains("unique") || msg.contains("duplicate key") { + transaction_timer.set_outcome("concurrency_violation"); + return Err(PersistenceError::ConcurrencyViolation { + expected: expected_sequence, + actual: new_seq, + }); + } else { + return Err(PersistenceError::Storage(msg)); + } + } + } + + if new_seq > expected_sequence { + segments::update_segment_after_append( + &mut tx, + tenant, + entity_type, + entity_id, + segment_index, + new_seq, + ) + .await?; + } + + // ADR-0153/0171: co-commit the entity's EXACT declared key set in THIS + // transaction (uniqueness was validated above). Empty current rows release + // every prior claim; deleting by entity also removes rows for declarations + // that no longer exist. + if reconciliation.keys { + crate::dbm::postgres_query!( + "DELETE FROM entity_key_index \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .execute(&mut *tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + for key in key_rows { + crate::dbm::postgres_query!( + "INSERT INTO entity_key_index \ + (tenant, entity_type, key_name, key_hash, entity_id, sequence_nr) \ + VALUES ($1, $2, $3, $4, $5, $6)", + ) + .bind(tenant) + .bind(entity_type) + .bind(&key.key_name) + .bind(&key.key_hash) + .bind(entity_id) + .bind(new_seq as i64) + .execute(&mut *tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + } + } + + // ADR-0155: co-commit the derived vector-index rows in THIS transaction. + // When the entity's type declares vector paths (`reconciliation.vectors`), DELETE + // all of the entity's rows first, then insert the current ones — so a delete + // transition or a cleared vector/model property (empty `vector_rows`) purges + // the stale rows instead of leaving them to rank forever. No uniqueness + // constraint; vectors are derived ranking state. + if reconciliation.vectors { + crate::dbm::postgres_query!( + "DELETE FROM entity_vector_index \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .execute(&mut *tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + for row in vector_rows { + crate::dbm::postgres_query!( + "INSERT INTO entity_vector_index \ + (tenant, entity_type, decl_name, model_tag, entity_id, vector, sequence_nr) \ + VALUES ($1, $2, $3, $4, $5, $6, $7)", + ) + .bind(tenant) + .bind(entity_type) + .bind(&row.decl_name) + .bind(&row.model_tag) + .bind(entity_id) + .bind(pack_f32_le(&row.vector)) + .bind(new_seq as i64) + .execute(&mut *tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + } + } + + if append_retires_snapshot( + expected_sequence, + events, + &reconciliation.snapshot_source, + entity_type, + entity_id, + ) { + crate::dbm::postgres_query!( + "DELETE FROM snapshots \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .execute(&mut *tx) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + } + if !events.is_empty() { + mark_query_projection_dirty(&mut tx, tenant, entity_type, entity_id).await?; + } + + let commit_started = Instant::now(); + tx.commit().await.map_err(|e| { + record_postgres_transaction_commit_duration( + commit_started.elapsed(), + EVENT_APPEND_OPERATION, + "error", + ); + PersistenceError::Storage(e.to_string()) + })?; + record_postgres_transaction_commit_duration( + commit_started.elapsed(), + EVENT_APPEND_OPERATION, + "ok", + ); + transaction_timer.set_outcome("ok"); + + Ok(new_seq) + } +} diff --git a/crates/temper-store-postgres/src/store/append_batch.rs b/crates/temper-store-postgres/src/store/append_batch.rs new file mode 100644 index 000000000..aa5f7da9d --- /dev/null +++ b/crates/temper-store-postgres/src/store/append_batch.rs @@ -0,0 +1,386 @@ +//! Atomic multi-stream PostgreSQL journal appends. + +use super::*; + +impl PostgresEventStore { + /// Atomically append to multiple entity journals in one PostgreSQL + /// transaction. Used as the storage foundation for cross-actor Composite + /// transactions: every stream's optimistic-concurrency and final declared-key + /// set must validate before any event is inserted. + pub(super) async fn append_batch_inner( + &self, + appends: &[PersistenceAppend], + ) -> Result, PersistenceError> { + if appends.is_empty() { + return Ok(Vec::new()); + } + + let mut lock_keys = std::collections::BTreeSet::new(); + let mut mutation_types = std::collections::BTreeSet::new(); + let mut unreconciled_types = std::collections::BTreeSet::new(); + let mut type_contracts = std::collections::BTreeMap::new(); + let mut batch_claim = None; + for append in appends { + if !append.reconcile_keys && !append.key_rows.is_empty() { + return Err(PersistenceError::Storage(format!( + "append_batch stream '{}' supplied key rows without exact reconciliation", + append.persistence_id + ))); + } + let (tenant, entity_type, entity_id) = + parse_persistence_id_parts(&append.persistence_id) + .map_err(PersistenceError::Storage)?; + if append.reconcile_keys || !append.events.is_empty() { + mutation_types.insert((tenant.to_string(), entity_type.to_string())); + } + if !append.events.is_empty() && !append.reconcile_keys { + unreconciled_types.insert((tenant.to_string(), entity_type.to_string())); + } + if append.reconcile_keys { + let type_key = (tenant.to_string(), entity_type.to_string()); + if let Some(existing) = type_contracts.get(&type_key) { + if existing != &append.key_set_signature { + return Err(PersistenceError::Storage(format!( + "append_batch supplied inconsistent key contracts for {tenant}:{entity_type}" + ))); + } + } else { + type_contracts.insert(type_key, append.key_set_signature.clone()); + } + } + let lock_key = event_stream_lock_key(tenant, entity_type, entity_id); + if !lock_keys.insert(lock_key) { + return Err(PersistenceError::Storage(format!( + "duplicate persistence stream '{}' in append_batch", + append.persistence_id + ))); + } + if let Some(claim) = &append.batch_idempotency + && batch_claim.replace(claim).is_some() + { + return Err(PersistenceError::Storage( + "append_batch supplied more than one idempotency claim".to_string(), + )); + } + if let Some(claim) = &append.batch_idempotency { + let (claim_tenant, claim_entity_type, claim_entity_id) = + parse_persistence_id_parts(&claim.persistence_id) + .map_err(PersistenceError::Storage)?; + lock_keys.insert(event_stream_lock_key( + claim_tenant, + claim_entity_type, + claim_entity_id, + )); + } + } + + let mut transaction_timer = PostgresTransactionTimer::start(EVENT_APPEND_OPERATION); + let acquire_started = Instant::now(); + let mut conn = match self.pool.acquire().await { + Ok(conn) => { + record_postgres_pool_acquire_duration( + acquire_started.elapsed(), + EVENT_APPEND_OPERATION, + "ok", + ); + conn + } + Err(e) => { + record_postgres_pool_acquire_duration( + acquire_started.elapsed(), + EVENT_APPEND_OPERATION, + "error", + ); + return Err(PersistenceError::Storage(e.to_string())); + } + }; + let begin_started = Instant::now(); + let mut tx = match conn.begin().await { + Ok(tx) => { + record_postgres_transaction_begin_duration( + begin_started.elapsed(), + EVENT_APPEND_OPERATION, + "ok", + ); + tx + } + Err(e) => { + record_postgres_transaction_begin_duration( + begin_started.elapsed(), + EVENT_APPEND_OPERATION, + "error", + ); + return Err(PersistenceError::Storage(e.to_string())); + } + }; + + for (tenant, entity_type) in &mutation_types { + lock_key_contract(&mut tx, tenant, entity_type).await?; + } + for lock_key in &lock_keys { + lock_event_stream(&mut tx, lock_key).await?; + } + + if let Some(claim) = batch_claim { + let existing: Option<(String,)> = crate::dbm::postgres_query_as!( + "SELECT intent_hash FROM persistence_batch_idempotency \ + WHERE persistence_id = $1 AND idempotency_key = $2", + ) + .bind(&claim.persistence_id) + .bind(&claim.idempotency_key) + .fetch_optional(&mut *tx) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + if let Some((committed_hash,)) = existing { + if committed_hash != claim.intent_hash { + return Err(PersistenceError::Storage(format!( + "atomic batch idempotency key '{}' was reused with a different intent", + claim.idempotency_key + ))); + } + let mut results = Vec::with_capacity(appends.len()); + for append in appends { + let (tenant, entity_type, entity_id) = + parse_persistence_id_parts(&append.persistence_id) + .map_err(PersistenceError::Storage)?; + let row: Option<(i64,)> = crate::dbm::postgres_query_as!( + "SELECT COALESCE(MAX(sequence_nr), 0) FROM events \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_optional(&mut *tx) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + results.push(PersistenceAppendResult { + persistence_id: append.persistence_id.clone(), + sequence_nr: row.map(|value| value.0 as u64).unwrap_or(0), + batch_already_applied: true, + }); + } + transaction_timer.set_outcome("idempotent_replay"); + return Ok(results); + } + } + + let mut parsed = Vec::with_capacity(appends.len()); + for append in appends { + let (tenant, entity_type, entity_id) = + parse_persistence_id_parts(&append.persistence_id) + .map_err(PersistenceError::Storage)?; + let row: Option<(i64,)> = crate::dbm::postgres_query_as!( + "SELECT COALESCE(MAX(sequence_nr), 0) FROM events \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_optional(&mut *tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + + let current_seq = row.map(|r| r.0 as u64).unwrap_or(0); + if current_seq != append.expected_sequence { + transaction_timer.set_outcome("concurrency_violation"); + return Err(PersistenceError::ConcurrencyViolation { + expected: append.expected_sequence, + actual: current_seq, + }); + } + if !matches!(&append.snapshot_source, SnapshotSourceFence::Unchecked) { + let current_snapshot = + load_snapshot_for_update(&mut tx, tenant, entity_type, entity_id).await?; + if !snapshot_source_matches(current_snapshot.as_ref(), &append.snapshot_source) { + return Err(PersistenceError::SnapshotGenerationChanged); + } + } + let segment_index = segments::open_segment_for_append( + &mut tx, + tenant, + entity_type, + entity_id, + current_seq, + ) + .await?; + parsed.push(( + tenant.to_string(), + entity_type.to_string(), + entity_id.to_string(), + segment_index, + )); + } + + for ((tenant, entity_type), signature) in &type_contracts { + reconcile_key_contract_state( + &mut tx, + tenant, + entity_type, + signature.as_deref(), + None, + KeyContractUse::LiveWrite, + ) + .await?; + } + for (tenant, entity_type) in &unreconciled_types { + invalidate_key_coverage_for_unreconciled_append(&mut tx, tenant, entity_type).await?; + } + + // ADR-0192: batch writes use the same exact declared-key contract as normal + // actor appends. Delete every participating entity's old rows first so an + // atomic batch may transfer ownership, then install each final set. This is + // deliberately before journal insertion; any uniqueness error rolls the + // entire transaction back without advancing any stream. + for (append, (tenant, entity_type, entity_id, _)) in appends.iter().zip(parsed.iter()) { + if !append.reconcile_keys { + continue; + } + crate::dbm::postgres_query!( + "DELETE FROM entity_key_index \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .execute(&mut *tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + } + for (append, (tenant, entity_type, entity_id, _)) in appends.iter().zip(parsed.iter()) { + if !append.reconcile_keys { + continue; + } + let final_sequence = append + .expected_sequence + .checked_add(append.events.len() as u64) + .ok_or_else(|| { + PersistenceError::Storage(format!( + "append_batch sequence overflow for '{}'", + append.persistence_id + )) + })?; + for key in &append.key_rows { + crate::dbm::postgres_query!( + "INSERT INTO entity_key_index \ + (tenant, entity_type, key_name, key_hash, entity_id, sequence_nr) \ + VALUES ($1, $2, $3, $4, $5, $6)", + ) + .bind(tenant) + .bind(entity_type) + .bind(&key.key_name) + .bind(&key.key_hash) + .bind(entity_id) + .bind(final_sequence as i64) + .execute(&mut *tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + } + } + + let mut results = Vec::with_capacity(appends.len()); + for (append, (tenant, entity_type, entity_id, segment_index)) in + appends.iter().zip(parsed.iter()) + { + let mut new_seq = append.expected_sequence; + for event in &append.events { + new_seq += 1; + let metadata_json = serde_json::to_value(&event.metadata) + .map_err(|e| PersistenceError::Serialization(e.to_string()))?; + + if let Err(e) = crate::dbm::postgres_query!( + "INSERT INTO events (tenant, entity_type, entity_id, sequence_nr, segment_index, event_type, payload, metadata) \ + VALUES ($1, $2, $3, $4, $5, $6, $7, $8)", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .bind(new_seq as i64) + .bind(*segment_index) + .bind(&event.event_type) + .bind(&event.payload) + .bind(metadata_json) + .execute(&mut *tx) + .await + { + let msg = e.to_string(); + if msg.contains("unique") || msg.contains("duplicate key") { + transaction_timer.set_outcome("concurrency_violation"); + return Err(PersistenceError::ConcurrencyViolation { + expected: append.expected_sequence, + actual: new_seq, + }); + } + return Err(PersistenceError::Storage(msg)); + } + } + if new_seq > append.expected_sequence { + segments::update_segment_after_append( + &mut tx, + tenant, + entity_type, + entity_id, + *segment_index, + new_seq, + ) + .await?; + } + if append_retires_snapshot( + append.expected_sequence, + &append.events, + &append.snapshot_source, + entity_type, + entity_id, + ) { + crate::dbm::postgres_query!( + "DELETE FROM snapshots \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .execute(&mut *tx) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + } + if !append.events.is_empty() { + mark_query_projection_dirty(&mut tx, tenant, entity_type, entity_id).await?; + } + results.push(PersistenceAppendResult { + persistence_id: append.persistence_id.clone(), + sequence_nr: new_seq, + batch_already_applied: false, + }); + } + + if let Some(claim) = batch_claim { + crate::dbm::postgres_query!( + "INSERT INTO persistence_batch_idempotency \ + (persistence_id, idempotency_key, intent_hash) VALUES ($1, $2, $3)", + ) + .bind(&claim.persistence_id) + .bind(&claim.idempotency_key) + .bind(&claim.intent_hash) + .execute(&mut *tx) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + } + + let commit_started = Instant::now(); + tx.commit().await.map_err(|e| { + record_postgres_transaction_commit_duration( + commit_started.elapsed(), + EVENT_APPEND_OPERATION, + "error", + ); + PersistenceError::Storage(e.to_string()) + })?; + record_postgres_transaction_commit_duration( + commit_started.elapsed(), + EVENT_APPEND_OPERATION, + "ok", + ); + transaction_timer.set_outcome("ok"); + + Ok(results) + } +} diff --git a/crates/temper-store-postgres/src/store/key_index.rs b/crates/temper-store-postgres/src/store/key_index.rs new file mode 100644 index 000000000..ec92e774a --- /dev/null +++ b/crates/temper-store-postgres/src/store/key_index.rs @@ -0,0 +1,406 @@ +//! PostgreSQL key-ownership reconciliation and coverage fencing. + +use sqlx::PgPool; +use temper_runtime::persistence::{EntityKeyRow, KeyIndexBackfillFence, PersistenceError}; + +mod activation; +mod coverage; +mod query; + +pub(super) use activation::{activate_contract, activate_contracts}; +pub(crate) use coverage::{ + DerivedWriteSource, KeyContractUse, invalidate_key_coverage_for_derived_write, + invalidate_key_coverage_for_unreconciled_append, reconcile_key_contract_state, +}; +pub(super) use query::{keyed_entity_ids, lookup}; + +/// Serialize every journal mutation and sequence-fenced derived-index repair for one +/// persistence stream. Advisory transaction locks avoid a new coordination table; +/// batch appends acquire them in sorted order to prevent deadlocks. +pub(crate) async fn lock_event_stream( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + persistence_id: &str, +) -> Result<(), PersistenceError> { + sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))") + .bind(persistence_id) + .execute(&mut **tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + Ok(()) +} + +pub(crate) fn event_stream_lock_key(tenant: &str, entity_type: &str, entity_id: &str) -> String { + format!("{tenant}:{entity_type}:{entity_id}") +} + +fn key_contract_lock_key(tenant: &str, entity_type: &str) -> String { + format!("key-index-contract:{tenant}:{entity_type}") +} + +/// Serialize type-wide key-contract revisions with live appends and conditional +/// watermark publication. Callers acquire this before any stream lock. +pub(crate) async fn lock_key_contract( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + tenant: &str, + entity_type: &str, +) -> Result<(), PersistenceError> { + lock_event_stream(tx, &key_contract_lock_key(tenant, entity_type)).await +} + +pub(super) async fn backfill_entity_keys( + pool: &PgPool, + tenant: &str, + entity_type: &str, + entity_id: &str, + expected_sequence: u64, + contract_fence: KeyIndexBackfillFence<'_>, + key_rows: &[EntityKeyRow], +) -> Result<(), PersistenceError> { + let mut tx = pool + .begin() + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + // Preserve the global lock order used by live appends and batches: the type + // contract fence is always acquired before an entity stream fence. + lock_key_contract(&mut tx, tenant, entity_type).await?; + let current_contract: Option<(String, i64)> = crate::dbm::postgres_query_as!( + "SELECT key_set, revision FROM key_index_contract_state \ + WHERE tenant = $1 AND entity_type = $2 FOR UPDATE", + ) + .bind(tenant) + .bind(entity_type) + .fetch_optional(&mut *tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + let actual_revision = current_contract + .as_ref() + .map(|(_, revision)| *revision as u64) + .unwrap_or(0); + if !matches!( + current_contract, + Some((ref signature, revision)) + if signature == contract_fence.key_set_signature + && revision as u64 == contract_fence.contract_revision + ) { + return Err(PersistenceError::KeyContractChanged { + expected_signature: contract_fence.key_set_signature.to_string(), + expected_revision: contract_fence.contract_revision, + actual_signature: current_contract.map(|(signature, _)| signature), + actual_revision, + }); + } + + let lock_key = event_stream_lock_key(tenant, entity_type, entity_id); + lock_event_stream(&mut tx, &lock_key).await?; + + let current_snapshot: Option<(i64, Vec)> = crate::dbm::postgres_query_as!( + "SELECT sequence_nr, state FROM snapshots \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_optional(&mut *tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + let snapshot_matches = match (contract_fence.expected_snapshot, current_snapshot.as_ref()) { + (None, None) => true, + (Some(expected), Some((sequence_nr, state))) => { + *sequence_nr >= 0 + && *sequence_nr as u64 == expected.sequence_nr + && state.as_slice() == expected.state + } + _ => false, + }; + if !snapshot_matches { + return Err(PersistenceError::SnapshotGenerationChanged); + } + + // Source authority is categorical, not numeric: journal first, then snapshot, + // then catalog only when neither stronger source exists. A higher compatibility + // catalog sequence must never outrank an exact snapshot generation. + let row: (i64, i64, bool) = crate::dbm::postgres_query_as!( + "SELECT CASE WHEN EXISTS ( \ + SELECT 1 FROM events \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ + ) THEN COALESCE(( \ + SELECT MAX(sequence_nr) FROM events \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ + ), 0) WHEN EXISTS ( \ + SELECT 1 FROM snapshots \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ + ) THEN COALESCE(( \ + SELECT sequence_nr FROM snapshots \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ + ), 0) ELSE COALESCE(( \ + SELECT sequence_nr FROM entity_catalog \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ + ), 0) \ + END, COALESCE(( \ + SELECT MAX(sequence_nr) FROM events \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ + ), 0), ( \ + ( \ + EXISTS ( \ + SELECT 1 FROM events \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ + ) \ + OR EXISTS ( \ + SELECT 1 FROM snapshots \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ + ) \ + OR EXISTS ( \ + SELECT 1 FROM entity_catalog \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ + ) \ + OR EXISTS ( \ + SELECT 1 FROM entity_field_index \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ + ) \ + ) \ + AND NOT EXISTS ( \ + SELECT 1 FROM events \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ + AND (CASE WHEN jsonb_typeof(payload) = 'object' AND payload ? 'to_status' \ + THEN payload ->> 'to_status' = 'Deleted' \ + ELSE event_type = 'Deleted' OR payload ->> 'action' = 'Deleted' \ + END) \ + ) \ + )", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_one(&mut *tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + let current_sequence = row.0 as u64; + let current_journal_sequence = row.1 as u64; + if current_journal_sequence != contract_fence.expected_journal_sequence { + return Err(PersistenceError::JournalBoundaryChanged { + expected: contract_fence.expected_journal_sequence, + actual: current_journal_sequence, + }); + } + let current_entity_live = row.2; + if current_entity_live != contract_fence.expected_entity_live { + return Err(PersistenceError::EntityLivenessChanged { + expected_live: contract_fence.expected_entity_live, + actual_live: current_entity_live, + }); + } + if current_sequence != expected_sequence { + return Err(PersistenceError::ConcurrencyViolation { + expected: expected_sequence, + actual: current_sequence, + }); + } + + for key in key_rows { + let holder: Option<(String,)> = crate::dbm::postgres_query_as!( + "SELECT entity_id FROM entity_key_index \ + WHERE tenant = $1 AND entity_type = $2 AND key_name = $3 AND key_hash = $4", + ) + .bind(tenant) + .bind(entity_type) + .bind(&key.key_name) + .bind(&key.key_hash) + .fetch_optional(&mut *tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + if let Some((existing,)) = holder + && existing != entity_id + { + return Err(PersistenceError::Storage(format!( + "duplicate declared key '{}' for {entity_type}: held by {existing}", + key.key_name + ))); + } + } + + // Exact reconciliation includes an empty current set, which releases every + // obsolete claim before the new rows are installed at the fenced sequence. + crate::dbm::postgres_query!( + "DELETE FROM entity_key_index \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .execute(&mut *tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + for key in key_rows { + crate::dbm::postgres_query!( + "INSERT INTO entity_key_index \ + (tenant, entity_type, key_name, key_hash, entity_id, sequence_nr) \ + VALUES ($1, $2, $3, $4, $5, $6)", + ) + .bind(tenant) + .bind(entity_type) + .bind(&key.key_name) + .bind(&key.key_hash) + .bind(entity_id) + .bind(expected_sequence as i64) + .execute(&mut *tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + } + tx.commit() + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + Ok(()) +} + +async fn upsert_watermark( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + tenant: &str, + entity_type: &str, + key_set: &str, +) -> Result<(), PersistenceError> { + crate::dbm::postgres_query!( + "INSERT INTO key_index_backfill_watermark (tenant, entity_type, key_set) \ + VALUES ($1, $2, $3) \ + ON CONFLICT (tenant, entity_type) \ + DO UPDATE SET key_set = EXCLUDED.key_set, completed_at = now()", + ) + .bind(tenant) + .bind(entity_type) + .bind(key_set) + .execute(&mut **tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + Ok(()) +} + +pub(super) async fn mark_backfilled( + pool: &PgPool, + tenant: &str, + entity_type: &str, + key_set: &str, +) -> Result<(), PersistenceError> { + let mut tx = pool + .begin() + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + lock_key_contract(&mut tx, tenant, entity_type).await?; + reconcile_key_contract_state( + &mut tx, + tenant, + entity_type, + Some(key_set), + None, + KeyContractUse::Backfill, + ) + .await?; + upsert_watermark(&mut tx, tenant, entity_type, key_set).await?; + tx.commit() + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + Ok(()) +} + +pub(super) async fn backfilled_types( + pool: &PgPool, + tenant: &str, +) -> Result, PersistenceError> { + let rows: Vec<(String, String)> = crate::dbm::postgres_query_as!( + "SELECT entity_type, key_set FROM key_index_backfill_watermark WHERE tenant = $1", + ) + .bind(tenant) + .fetch_all(pool) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + Ok(rows) +} + +pub(super) async fn activated_contracts( + pool: &PgPool, +) -> Result, PersistenceError> { + let rows = crate::dbm::postgres_query_as!( + "SELECT tenant, entity_type FROM key_index_contract_state \ + WHERE activated_key_set IS NOT NULL ORDER BY tenant, entity_type", + ) + .fetch_all(pool) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + Ok(rows) +} + +pub(super) async fn reconciliation_revision( + pool: &PgPool, + tenant: &str, + entity_type: &str, +) -> Result { + let revision: Option = crate::dbm::postgres_query_scalar!( + "SELECT revision FROM key_index_contract_state \ + WHERE tenant = $1 AND entity_type = $2", + ) + .bind(tenant) + .bind(entity_type) + .fetch_optional(pool) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + Ok(revision.unwrap_or(0) as u64) +} + +pub(super) async fn begin_backfill( + pool: &PgPool, + tenant: &str, + entity_type: &str, + key_set: &str, +) -> Result { + let mut tx = pool + .begin() + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + lock_key_contract(&mut tx, tenant, entity_type).await?; + let revision = reconcile_key_contract_state( + &mut tx, + tenant, + entity_type, + Some(key_set), + None, + KeyContractUse::Backfill, + ) + .await?; + tx.commit() + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + Ok(revision) +} + +pub(super) async fn mark_backfilled_if_revision( + pool: &PgPool, + tenant: &str, + entity_type: &str, + key_set: &str, + expected_revision: u64, +) -> Result { + let mut tx = pool + .begin() + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + lock_key_contract(&mut tx, tenant, entity_type).await?; + let current: Option<(String, i64)> = crate::dbm::postgres_query_as!( + "SELECT key_set, revision FROM key_index_contract_state \ + WHERE tenant = $1 AND entity_type = $2 FOR UPDATE", + ) + .bind(tenant) + .bind(entity_type) + .fetch_optional(&mut *tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + if !matches!( + current, + Some((ref current_key_set, revision)) + if current_key_set == key_set && revision as u64 == expected_revision + ) { + return Ok(false); + } + upsert_watermark(&mut tx, tenant, entity_type, key_set).await?; + tx.commit() + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + Ok(true) +} diff --git a/crates/temper-store-postgres/src/store/key_index/activation.rs b/crates/temper-store-postgres/src/store/key_index/activation.rs new file mode 100644 index 000000000..f83f14414 --- /dev/null +++ b/crates/temper-store-postgres/src/store/key_index/activation.rs @@ -0,0 +1,128 @@ +use sqlx::PgPool; +use temper_runtime::persistence::{KeyContractActivation, PersistenceError}; + +use super::{KeyContractUse, lock_key_contract, reconcile_key_contract_state}; + +pub(in crate::store) async fn activate_contract( + pool: &PgPool, + tenant: &str, + entity_type: &str, + key_set: &str, + purge_existing_rows: bool, +) -> Result { + let mut tx = pool + .begin() + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + lock_key_contract(&mut tx, tenant, entity_type).await?; + let prior_activation: Option<(Option, Option)> = + crate::dbm::postgres_query_as!( + "SELECT activated_key_set, activated_spec_fingerprint \ + FROM key_index_contract_state \ + WHERE tenant = $1 AND entity_type = $2", + ) + .bind(tenant) + .bind(entity_type) + .fetch_optional(&mut *tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + let semantic_contract_changed = !matches!( + prior_activation, + Some((Some(ref active_key_set), Some(ref active_fingerprint))) + if active_key_set == key_set && active_fingerprint == key_set + ); + let revision = reconcile_key_contract_state( + &mut tx, + tenant, + entity_type, + Some(key_set), + Some(key_set), + KeyContractUse::Activation, + ) + .await?; + if purge_existing_rows || semantic_contract_changed { + crate::dbm::postgres_query!( + "DELETE FROM entity_key_index WHERE tenant = $1 AND entity_type = $2", + ) + .bind(tenant) + .bind(entity_type) + .execute(&mut *tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + } + tx.commit() + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + Ok(revision) +} + +pub(in crate::store) async fn activate_contracts( + pool: &PgPool, + tenant: &str, + activations: &[KeyContractActivation], +) -> Result, PersistenceError> { + let mut ordered = activations.iter().collect::>(); + ordered.sort_by(|left, right| left.entity_type.cmp(&right.entity_type)); + if ordered + .windows(2) + .any(|pair| pair[0].entity_type == pair[1].entity_type) + { + return Err(PersistenceError::Storage(format!( + "duplicate key activation in tenant {tenant}" + ))); + } + + let mut tx = pool + .begin() + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + // Acquire every type lock in deterministic order before mutating any row, + // making the tenant activation all-or-nothing and deadlock-safe. + for activation in &ordered { + lock_key_contract(&mut tx, tenant, &activation.entity_type).await?; + } + let mut epochs = std::collections::BTreeMap::new(); + for activation in ordered { + let prior_activation: Option<(Option, Option)> = + crate::dbm::postgres_query_as!( + "SELECT activated_key_set, activated_spec_fingerprint \ + FROM key_index_contract_state \ + WHERE tenant = $1 AND entity_type = $2", + ) + .bind(tenant) + .bind(&activation.entity_type) + .fetch_optional(&mut *tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + let semantic_contract_changed = !matches!( + prior_activation, + Some((Some(ref active_key_set), Some(ref active_fingerprint))) + if active_key_set == &activation.key_set + && active_fingerprint == &activation.spec_fingerprint + ); + let epoch = reconcile_key_contract_state( + &mut tx, + tenant, + &activation.entity_type, + Some(&activation.key_set), + Some(&activation.spec_fingerprint), + KeyContractUse::Activation, + ) + .await?; + if activation.purge_existing_rows || semantic_contract_changed { + crate::dbm::postgres_query!( + "DELETE FROM entity_key_index WHERE tenant = $1 AND entity_type = $2", + ) + .bind(tenant) + .bind(&activation.entity_type) + .execute(&mut *tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + } + epochs.insert(activation.entity_type.clone(), epoch); + } + tx.commit() + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + Ok(epochs) +} diff --git a/crates/temper-store-postgres/src/store/key_index/coverage.rs b/crates/temper-store-postgres/src/store/key_index/coverage.rs new file mode 100644 index 000000000..216e59afd --- /dev/null +++ b/crates/temper-store-postgres/src/store/key_index/coverage.rs @@ -0,0 +1,353 @@ +//! Type-wide key-coverage contract and durable-source invalidation. + +use temper_runtime::persistence::{PersistenceError, decode_activated_key_contract}; + +const UNKNOWN_KEY_SET_SIGNATURE: &str = ""; + +type KeyContractStateRow = (String, i64, Option, Option, i64, bool); + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum KeyContractUse { + LiveWrite, + Backfill, + Activation, +} + +/// Identifies the durable derived-state writer that can invalidate coverage. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum DerivedWriteSource { + /// Snapshot bytes contribute the recovery baseline even when the journal has + /// reached the same or a later sequence. + Snapshot, + /// Query-plane catalog state is authoritative only when neither a journal nor + /// a snapshot exists for the entity. + Catalog, +} + +/// Record the key signature used by a durable write. A changed or previously +/// unknown contract advances the monotonic revision and invalidates coverage in the +/// same transaction as the journal append. +pub(crate) async fn reconcile_key_contract_state( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + tenant: &str, + entity_type: &str, + key_set_signature: Option<&str>, + activation_fingerprint: Option<&str>, + contract_use: KeyContractUse, +) -> Result { + let raw_contract = key_set_signature.unwrap_or(UNKNOWN_KEY_SET_SIGNATURE); + let (supplied, attempted_epoch) = decode_activated_key_contract(raw_contract); + let existing: Option = + crate::dbm::postgres_query_as!( + "SELECT key_set, revision, activated_key_set, activated_spec_fingerprint, activation_epoch, \ + EXISTS(SELECT 1 FROM key_index_backfill_watermark watermark \ + WHERE watermark.tenant = $1 \ + AND watermark.entity_type = $2 \ + AND watermark.key_set = key_index_contract_state.key_set) \ + FROM key_index_contract_state \ + WHERE tenant = $1 AND entity_type = $2 FOR UPDATE", + ) + .bind(tenant) + .bind(entity_type) + .fetch_optional(&mut **tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + + if let Some(( + current, + revision, + prior_activation, + prior_fingerprint, + activation_epoch, + activation_ready, + )) = existing + { + if let Some(active) = prior_activation.as_deref() + && active != supplied + && contract_use != KeyContractUse::Activation + { + return Err(PersistenceError::KeyContractNotActive { + activated_signature: active.to_string(), + attempted_signature: supplied.to_string(), + }); + } + if contract_use == KeyContractUse::LiveWrite + && prior_activation.is_some() + && attempted_epoch != Some(activation_epoch as u64) + { + return Err(PersistenceError::KeyContractActivationStale { + activated_epoch: activation_epoch as u64, + attempted_epoch, + }); + } + if contract_use == KeyContractUse::LiveWrite + && prior_activation.is_some() + && !activation_ready + { + return Err(PersistenceError::KeyContractActivationNotReady { + activated_epoch: activation_epoch as u64, + activated_signature: prior_activation + .clone() + .expect("activation presence checked"), + }); + } + let activation_fingerprint = activation_fingerprint.unwrap_or(supplied); + let semantic_contract_changed = contract_use == KeyContractUse::Activation + && (prior_activation.as_deref() != Some(supplied) + || prior_fingerprint.as_deref() != Some(activation_fingerprint)); + let (activated_key_set, activated_spec_fingerprint, next_activation_epoch) = + if contract_use == KeyContractUse::Activation { + let next = activation_epoch.checked_add(1).ok_or_else(|| { + PersistenceError::Storage(format!( + "key activation epoch overflow for {tenant}:{entity_type}" + )) + })?; + ( + Some(supplied.to_string()), + Some(activation_fingerprint.to_string()), + next, + ) + } else { + ( + prior_activation.clone(), + prior_fingerprint.clone(), + activation_epoch, + ) + }; + if current == supplied { + let next_revision = if semantic_contract_changed { + revision.checked_add(1).ok_or_else(|| { + PersistenceError::Storage(format!( + "key contract revision overflow for {tenant}:{entity_type}" + )) + })? + } else { + revision + }; + if activated_key_set != prior_activation || next_activation_epoch != activation_epoch { + crate::dbm::postgres_query!( + "UPDATE key_index_contract_state \ + SET activated_key_set = $3, activated_spec_fingerprint = $4, \ + activation_epoch = $5, revision = $6, updated_at = now() \ + WHERE tenant = $1 AND entity_type = $2", + ) + .bind(tenant) + .bind(entity_type) + .bind(activated_key_set) + .bind(activated_spec_fingerprint) + .bind(next_activation_epoch) + .bind(next_revision) + .execute(&mut **tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + } + if semantic_contract_changed { + crate::dbm::postgres_query!( + "DELETE FROM key_index_backfill_watermark \ + WHERE tenant = $1 AND entity_type = $2", + ) + .bind(tenant) + .bind(entity_type) + .execute(&mut **tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + } + return Ok(if contract_use == KeyContractUse::Activation { + next_activation_epoch as u64 + } else { + revision as u64 + }); + } + let next = revision.checked_add(1).ok_or_else(|| { + PersistenceError::Storage(format!( + "key contract revision overflow for {tenant}:{entity_type}" + )) + })?; + crate::dbm::postgres_query!( + "UPDATE key_index_contract_state \ + SET key_set = $3, revision = $4, activated_key_set = $5, \ + activated_spec_fingerprint = $6, activation_epoch = $7, updated_at = now() \ + WHERE tenant = $1 AND entity_type = $2", + ) + .bind(tenant) + .bind(entity_type) + .bind(supplied) + .bind(next) + .bind(activated_key_set) + .bind(activated_spec_fingerprint) + .bind(next_activation_epoch) + .execute(&mut **tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + crate::dbm::postgres_query!( + "DELETE FROM key_index_backfill_watermark \ + WHERE tenant = $1 AND entity_type = $2", + ) + .bind(tenant) + .bind(entity_type) + .execute(&mut **tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + return Ok(if contract_use == KeyContractUse::Activation { + next_activation_epoch as u64 + } else { + next as u64 + }); + } + + let (activated_key_set, activated_spec_fingerprint, activation_epoch) = match contract_use { + KeyContractUse::Activation => ( + Some(supplied), + Some(activation_fingerprint.unwrap_or(supplied)), + 1_i64, + ), + KeyContractUse::Backfill | KeyContractUse::LiveWrite => (None, None, 0_i64), + }; + crate::dbm::postgres_query!( + "INSERT INTO key_index_contract_state \ + (tenant, entity_type, key_set, revision, activated_key_set, \ + activated_spec_fingerprint, activation_epoch) \ + VALUES ($1, $2, $3, 1, $4, $5, $6)", + ) + .bind(tenant) + .bind(entity_type) + .bind(supplied) + .bind(activated_key_set) + .bind(activated_spec_fingerprint) + .bind(activation_epoch) + .execute(&mut **tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + // A legacy watermark without contract state cannot prove that intervening writes + // used the same definition. Withhold it until one fenced repair completes. + crate::dbm::postgres_query!( + "DELETE FROM key_index_backfill_watermark \ + WHERE tenant = $1 AND entity_type = $2", + ) + .bind(tenant) + .bind(entity_type) + .execute(&mut **tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + Ok(1) +} + +/// Advance the current reconciliation epoch when a changed durable snapshot or +/// non-journal-dominated catalog mutation can alter ownership recovery. +/// +/// Callers hold the type contract lock and the entity stream lock. The signature +/// remains unchanged; only the monotonic revision advances. Snapshot bytes are a +/// semantic recovery baseline and therefore always invalidate when changed. A +/// catalog mutation is compatibility-only whenever a journal or snapshot exists +/// and can reuse that stronger source's fence. +pub(crate) async fn invalidate_key_coverage_for_derived_write( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + tenant: &str, + entity_type: &str, + entity_id: &str, + source: DerivedWriteSource, +) -> Result<(), PersistenceError> { + if source == DerivedWriteSource::Catalog { + let stronger_source_exists: bool = crate::dbm::postgres_query_scalar!( + "SELECT EXISTS(SELECT 1 FROM events \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3) \ + OR EXISTS(SELECT 1 FROM snapshots \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3)", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_one(&mut **tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + if stronger_source_exists { + return Ok(()); + } + } + + let current: Option<(String, i64)> = crate::dbm::postgres_query_as!( + "SELECT key_set, revision FROM key_index_contract_state \ + WHERE tenant = $1 AND entity_type = $2 FOR UPDATE", + ) + .bind(tenant) + .bind(entity_type) + .fetch_optional(&mut **tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + let Some((_, revision)) = current else { + return Ok(()); + }; + let next = revision.checked_add(1).ok_or_else(|| { + PersistenceError::Storage(format!( + "key reconciliation revision overflow for {tenant}:{entity_type}" + )) + })?; + crate::dbm::postgres_query!( + "UPDATE key_index_contract_state \ + SET revision = $3, updated_at = now() \ + WHERE tenant = $1 AND entity_type = $2", + ) + .bind(tenant) + .bind(entity_type) + .bind(next) + .execute(&mut **tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + crate::dbm::postgres_query!( + "DELETE FROM key_index_backfill_watermark \ + WHERE tenant = $1 AND entity_type = $2", + ) + .bind(tenant) + .bind(entity_type) + .execute(&mut **tx) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + Ok(()) +} + +/// An append that does not carry exact declared-key rows cannot preserve an +/// activated ownership proof. Advance the revision and close readiness in the +/// same transaction so a racing repair CAS loses before the journal mutates. +pub(crate) async fn invalidate_key_coverage_for_unreconciled_append( + tx: &mut sqlx::Transaction<'_, sqlx::Postgres>, + tenant: &str, + entity_type: &str, +) -> Result<(), PersistenceError> { + let current: Option<(i64,)> = crate::dbm::postgres_query_as!( + "SELECT revision FROM key_index_contract_state \ + WHERE tenant = $1 AND entity_type = $2 FOR UPDATE", + ) + .bind(tenant) + .bind(entity_type) + .fetch_optional(&mut **tx) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + let Some((revision,)) = current else { + return Ok(()); + }; + let next = revision.checked_add(1).ok_or_else(|| { + PersistenceError::Storage(format!( + "key reconciliation revision overflow for {tenant}:{entity_type}" + )) + })?; + crate::dbm::postgres_query!( + "UPDATE key_index_contract_state SET revision = $3, updated_at = now() \ + WHERE tenant = $1 AND entity_type = $2", + ) + .bind(tenant) + .bind(entity_type) + .bind(next) + .execute(&mut **tx) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + crate::dbm::postgres_query!( + "DELETE FROM key_index_backfill_watermark \ + WHERE tenant = $1 AND entity_type = $2", + ) + .bind(tenant) + .bind(entity_type) + .execute(&mut **tx) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + Ok(()) +} diff --git a/crates/temper-store-postgres/src/store/key_index/query.rs b/crates/temper-store-postgres/src/store/key_index/query.rs new file mode 100644 index 000000000..a3243e737 --- /dev/null +++ b/crates/temper-store-postgres/src/store/key_index/query.rs @@ -0,0 +1,45 @@ +//! Declared-key ownership queries. + +use sqlx::PgPool; +use temper_runtime::persistence::{EntityKeyLookup, PersistenceError}; + +pub(in crate::store) async fn keyed_entity_ids( + pool: &PgPool, + tenant: &str, + entity_type: &str, +) -> Result, PersistenceError> { + let rows: Vec<(String,)> = crate::dbm::postgres_query_as!( + "SELECT DISTINCT entity_id FROM entity_key_index \ + WHERE tenant = $1 AND entity_type = $2", + ) + .bind(tenant) + .bind(entity_type) + .fetch_all(pool) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + Ok(rows.into_iter().map(|(entity_id,)| entity_id).collect()) +} + +pub(in crate::store) async fn lookup( + pool: &PgPool, + tenant: &str, + entity_type: &str, + key_name: &str, + key_hash: &str, +) -> Result, PersistenceError> { + let row: Option<(String, i64)> = crate::dbm::postgres_query_as!( + "SELECT entity_id, sequence_nr FROM entity_key_index \ + WHERE tenant = $1 AND entity_type = $2 AND key_name = $3 AND key_hash = $4", + ) + .bind(tenant) + .bind(entity_type) + .bind(key_name) + .bind(key_hash) + .fetch_optional(pool) + .await + .map_err(|e| PersistenceError::Storage(e.to_string()))?; + Ok(row.map(|(entity_id, sequence_nr)| EntityKeyLookup { + entity_id, + sequence_nr: sequence_nr as u64, + })) +} diff --git a/crates/temper-store-postgres/src/store/snapshot.rs b/crates/temper-store-postgres/src/store/snapshot.rs new file mode 100644 index 000000000..c9b8b6c1f --- /dev/null +++ b/crates/temper-store-postgres/src/store/snapshot.rs @@ -0,0 +1,150 @@ +//! Source-fenced PostgreSQL snapshot persistence. + +use super::*; + +impl PostgresEventStore { + pub(super) async fn save_snapshot_inner( + &self, + persistence_id: &str, + sequence_nr: u64, + snapshot: &[u8], + source: &SnapshotSourceFence, + key_contract: Option<&str>, + ) -> Result<(), PersistenceError> { + let (tenant, entity_type, entity_id) = + parse_persistence_id_parts(persistence_id).map_err(PersistenceError::Storage)?; + let sequence_nr = i64::try_from(sequence_nr).map_err(|_| { + PersistenceError::Storage("snapshot sequence exceeds PostgreSQL bigint".to_string()) + })?; + + let mut tx = self + .pool + .begin() + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + + lock_key_contract(&mut tx, tenant, entity_type).await?; + let lock_key = event_stream_lock_key(tenant, entity_type, entity_id); + lock_event_stream(&mut tx, &lock_key).await?; + if let Some(key_contract) = key_contract { + reconcile_key_contract_state( + &mut tx, + tenant, + entity_type, + Some(key_contract), + None, + KeyContractUse::LiveWrite, + ) + .await?; + } + + let previous = load_snapshot_for_update(&mut tx, tenant, entity_type, entity_id).await?; + if !snapshot_source_matches(previous.as_ref(), source) { + return Err(PersistenceError::SnapshotGenerationChanged); + } + let journal_generation: (i64, Option, Option) = + crate::dbm::postgres_query_as!( + "SELECT COALESCE(MAX(sequence_nr), 0), \ + (SELECT event_type FROM events \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 AND sequence_nr = 1), \ + (SELECT payload FROM events \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 AND sequence_nr = 1) \ + FROM events \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_one(&mut *tx) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + if matches!(source, SnapshotSourceFence::Unchecked) + && journal_generation.0 > 0 + && journal_generation.2.as_ref().is_some_and(|payload| { + is_state_materialization_payload_for( + journal_generation.1.as_deref().unwrap_or_default(), + payload, + entity_type, + entity_id, + ) + }) + { + tx.commit() + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + return Ok(()); + } + let snapshot_is_noop = previous.as_ref().is_some_and(|(stored_sequence, stored)| { + *stored_sequence > sequence_nr + || (*stored_sequence == sequence_nr && stored.as_slice() == snapshot) + }); + if snapshot_is_noop { + tx.commit() + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + return Ok(()); + } + let same_sequence_replacement = previous + .as_ref() + .is_some_and(|(stored_sequence, _)| *stored_sequence == sequence_nr); + + if key_contract.is_none() { + invalidate_key_coverage_for_derived_write( + &mut tx, + tenant, + entity_type, + entity_id, + DerivedWriteSource::Snapshot, + ) + .await?; + } + + crate::dbm::postgres_query!( + "INSERT INTO snapshots (tenant, entity_type, entity_id, sequence_nr, state) \ + VALUES ($1, $2, $3, $4, $5) \ + ON CONFLICT (tenant, entity_type, entity_id) \ + DO UPDATE SET sequence_nr = $4, state = $5, created_at = now()", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .bind(sequence_nr) + .bind(snapshot) + .execute(&mut *tx) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + + crate::dbm::postgres_query!( + "INSERT INTO snapshot_history (tenant, entity_type, entity_id, sequence_nr, state) \ + VALUES ($1, $2, $3, $4, $5) \ + ON CONFLICT (tenant, entity_type, entity_id, sequence_nr) \ + DO UPDATE SET state = $5, created_at = now()", + ) + .bind(tenant) + .bind(entity_type) + .bind(entity_id) + .bind(sequence_nr) + .bind(snapshot) + .execute(&mut *tx) + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + + if !same_sequence_replacement { + segments::rotate_after_snapshot( + &mut tx, + tenant, + entity_type, + entity_id, + sequence_nr as u64, + ) + .await?; + } + + mark_query_projection_dirty(&mut tx, tenant, entity_type, entity_id).await?; + + tx.commit() + .await + .map_err(|error| PersistenceError::Storage(error.to_string()))?; + Ok(()) + } +} diff --git a/crates/temper-store-postgres/src/store/tests.rs b/crates/temper-store-postgres/src/store/tests.rs new file mode 100644 index 000000000..2ae8b8b8d --- /dev/null +++ b/crates/temper-store-postgres/src/store/tests.rs @@ -0,0 +1,440 @@ +//! PostgreSQL event-store regressions. + +use super::*; +use crate::migration::run_migrations; +use temper_runtime::tenant::parse_persistence_id_parts; + +fn parse_pid(persistence_id: &str) -> Result<(&str, &str, &str), PersistenceError> { + parse_persistence_id_parts(persistence_id).map_err(PersistenceError::Storage) +} + +fn test_envelope(event_type: &str, payload: serde_json::Value) -> PersistenceEnvelope { + PersistenceEnvelope { + sequence_nr: 0, + event_type: event_type.to_string(), + payload, + metadata: EventMetadata { + event_id: uuid::Uuid::new_v4(), + causation_id: uuid::Uuid::new_v4(), + correlation_id: uuid::Uuid::new_v4(), + timestamp: chrono::Utc::now(), + actor_id: "store-test".to_string(), + }, + } +} + +// -- persistence_id parsing --------------------------------------------- + +#[test] +fn parse_3_segment_persistence_id() { + let (tenant, entity_type, entity_id) = + parse_pid("alpha:Order:abc-123").expect("valid three-segment persistence ID"); + assert_eq!(tenant, "alpha"); + assert_eq!(entity_type, "Order"); + assert_eq!(entity_id, "abc-123"); +} + +#[test] +fn parse_legacy_2_segment_persistence_id() { + let (tenant, entity_type, entity_id) = + parse_pid("Order:abc-123").expect("valid legacy two-segment persistence ID"); + assert_eq!(tenant, "default"); + assert_eq!(entity_type, "Order"); + assert_eq!(entity_id, "abc-123"); +} + +#[test] +fn parse_3_segment_with_colons_in_id() { + // splitn(3, ':') puts everything after the second colon into entity_id + let (tenant, entity_type, entity_id) = + parse_pid("beta:Task:T-1:sub").expect("valid persistence ID with colon in entity ID"); + assert_eq!(tenant, "beta"); + assert_eq!(entity_type, "Task"); + assert_eq!(entity_id, "T-1:sub"); +} + +#[test] +fn parse_persistence_id_missing_colon() { + let err = parse_pid("OrderAbc123").unwrap_err(); + assert!( + matches!(err, PersistenceError::Storage(_)), + "expected Storage error, got: {err:?}" + ); +} + +#[test] +fn parse_persistence_id_empty_segment() { + assert!(parse_pid(":Order:abc").is_err()); + assert!(parse_pid("tenant::abc").is_err()); + assert!(parse_pid("tenant:Order:").is_err()); + assert!(parse_pid(":abc").is_err()); + assert!(parse_pid("Order:").is_err()); +} + +#[test] +fn list_entity_ids_returns_distinct_pairs() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => { + eprintln!("skipping Postgres integration test: DATABASE_URL is not set"); + return; + } + }; + + sqlx::test_block_on(async { + let pool = PgPool::connect(&database_url) + .await + .expect("connect to DATABASE_URL"); + run_migrations(&pool).await.expect("run migrations"); + let store = PostgresEventStore::new(pool); + + let tenant_a = format!("tenant-a-{}", uuid::Uuid::new_v4()); + let tenant_b = format!("tenant-b-{}", uuid::Uuid::new_v4()); + + let order_1 = format!("{tenant_a}:Order:ord-1"); + let order_2 = format!("{tenant_a}:Order:ord-2"); + let task_1 = format!("{tenant_a}:Task:task-1"); + let other_tenant = format!("{tenant_b}:Order:ord-9"); + + store + .append( + &order_1, + 0, + &[test_envelope( + "OrderCreated", + serde_json::json!({"id":"ord-1"}), + )], + ) + .await + .expect("append first order event"); + store + .append( + &order_1, + 1, + &[test_envelope( + "OrderUpdated", + serde_json::json!({"step": 2}), + )], + ) + .await + .expect("append order update event"); + store + .append( + &order_2, + 0, + &[test_envelope( + "OrderCreated", + serde_json::json!({"id":"ord-2"}), + )], + ) + .await + .expect("append second order event"); + store + .append( + &task_1, + 0, + &[test_envelope( + "TaskCreated", + serde_json::json!({"id":"task-1"}), + )], + ) + .await + .expect("append task event"); + store + .append( + &other_tenant, + 0, + &[test_envelope( + "OrderCreated", + serde_json::json!({"id":"ord-9"}), + )], + ) + .await + .expect("append other tenant event"); + + let mut entities = store + .list_entity_ids(&tenant_a) + .await + .expect("list tenant entity IDs"); + entities.sort(); + + assert_eq!( + entities, + vec![ + ("Order".to_string(), "ord-1".to_string()), + ("Order".to_string(), "ord-2".to_string()), + ("Task".to_string(), "task-1".to_string()), + ] + ); + }); +} + +#[test] +fn postgres_platform_methods_are_part_of_the_store_surface() { + // Compile-only check: the function body is never executed, so the + // unawaited futures are intentional. Clippy's let_underscore_future + // lint catches forgotten awaits in real code; here it's a false + // positive. + #[expect( + clippy::let_underscore_future, + reason = "compile-only API coverage intentionally constructs but never polls futures" + )] + fn assert_methods(store: &PostgresEventStore) { + let _ = store.upsert_query_projection( + "tenant", + "Session", + "s-1", + "Running", + &serde_json::json!({"phase":"Running"}), + 1, + ); + let _ = store.remove_query_projection("tenant", "Session", "s-1"); + let _ = store.query_field_index( + "tenant", + "Session", + "fields @> $3::jsonb", + vec!["{\"phase\":\"Running\"}".to_string()], + ); + let _ = store.persist_trajectory(crate::PostgresTrajectoryInsert { + tenant: "tenant", + entity_type: "Session", + entity_id: "s-1", + action: "ProgressMade", + success: true, + from_status: Some("Running"), + to_status: Some("Running"), + error: None, + agent_id: Some("agent"), + session_id: Some("session"), + authz_denied: None, + denied_resource: None, + denied_module: None, + source: Some("Entity"), + spec_governed: Some(true), + created_at: "2026-04-28T00:00:00Z", + request_body: Some("{\"ok\":true}"), + intent: Some("test"), + matched_policy_ids: Some("[\"policy:test\"]"), + }); + let _ = store.save_policy( + "tenant", + "primary", + "permit(principal, action, resource);", + "test", + ); + let _ = store.load_policies_for_tenant("tenant"); + let _ = store.load_all_policies(); + let _ = store.toggle_policy_enabled("tenant", "primary", true); + let _ = store.update_policy_text( + "tenant", + "primary", + "permit(principal, action, resource);", + "test", + ); + let _ = store.delete_policy("tenant", "primary"); + } + + let _ = assert_methods; +} + +#[test] +fn postgres_long_tail_methods_are_part_of_the_store_surface() { + let _ = PostgresEventStore::load_recent_trajectories; + let _ = PostgresEventStore::load_unmet_intent_rows; + let _ = PostgresEventStore::load_submit_spec_timestamps; + let _ = PostgresEventStore::count_trajectories_by_tenant; + let _ = PostgresEventStore::query_trajectory_stats; + let _ = PostgresEventStore::query_trajectories_by_agent; + let _ = PostgresEventStore::query_agent_summaries; + + let _ = PostgresEventStore::upsert_feature_request; + let _ = PostgresEventStore::list_feature_requests; + let _ = PostgresEventStore::update_feature_request; + let _ = PostgresEventStore::insert_evolution_record; + let _ = PostgresEventStore::get_evolution_record; + let _ = PostgresEventStore::list_evolution_records; + let _ = PostgresEventStore::list_ranked_insights; + let _ = PostgresEventStore::insert_design_time_event; + let _ = PostgresEventStore::list_design_time_events; + + let _ = PostgresEventStore::persist_ots_trajectory; + let _ = PostgresEventStore::list_ots_trajectories; + let _ = PostgresEventStore::get_ots_trajectory; + + let _ = PostgresEventStore::put_blob; + let _ = PostgresEventStore::put_blob_with_ttl; + let _ = PostgresEventStore::get_blob; + let _ = PostgresEventStore::sweep_expired_blobs; + + let _ = PostgresEventStore::upsert_secret; + let _ = PostgresEventStore::delete_secret; + let _ = PostgresEventStore::load_secrets_for_tenant; + + let _ = PostgresEventStore::upsert_policy_denial_pattern; + let _ = PostgresEventStore::load_policy_denial_patterns; + + let _ = PostgresEventStore::query_decisions; + let _ = PostgresEventStore::query_all_decisions; + let _ = PostgresEventStore::get_pending_decision; + + let _ = PostgresEventStore::load_wasm_module; + let _ = PostgresEventStore::load_wasm_module_metadata_all_tenants; + let _ = PostgresEventStore::persist_wasm_invocation; + let _ = PostgresEventStore::load_recent_wasm_invocations; + let _ = PostgresEventStore::delete_wasm_module; + + let _ = PostgresEventStore::upsert_published_artifact; + let _ = PostgresEventStore::load_published_artifact; +} + +#[test] +fn postgres_query_projection_batch_method_is_part_of_the_store_surface() { + let _ = PostgresEventStore::load_query_projection_fields_many; + let _ = PostgresEventStore::load_selected_entity_catalog_rows_pg; +} + +#[test] +fn load_query_projection_fields_many_returns_requested_fields() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => { + eprintln!("skipping Postgres integration test: DATABASE_URL is not set"); + return; + } + }; + + sqlx::test_block_on(async { + let pool = PgPool::connect(&database_url) + .await + .expect("connect to DATABASE_URL"); + run_migrations(&pool).await.expect("run migrations"); + let store = PostgresEventStore::new(pool.clone()); + let tenant = format!("tenant-projection-{}", uuid::Uuid::new_v4()); + + store + .upsert_query_projection( + &tenant, + "File", + "file-a", + "Ready", + &serde_json::json!({ + "content_hash": "sha256:file-a", + "mime_type": "text/plain", + "has_content": true, + }), + 12, + ) + .await + .expect("upsert query projection row"); + + let rows = store + .load_query_projection_fields_many( + &tenant, + "File", + &["file-a".to_string(), "missing".to_string()], + &["content_hash", "has_content"], + ) + .await + .expect("load requested projection fields"); + + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].entity_id, "file-a"); + assert_eq!(rows[0].status, "Ready"); + assert_eq!( + rows[0] + .fields + .get("content_hash") + .and_then(|value| value.as_deref()), + Some("sha256:file-a") + ); + assert_eq!( + rows[0] + .fields + .get("has_content") + .and_then(|value| value.as_deref()), + Some("true") + ); + + crate::dbm::postgres_query!("DELETE FROM entity_field_index WHERE tenant = $1") + .bind(&tenant) + .execute(&pool) + .await + .expect("delete test entity field index rows"); + crate::dbm::postgres_query!("DELETE FROM entity_catalog WHERE tenant = $1") + .bind(&tenant) + .execute(&pool) + .await + .expect("delete test entity catalog rows"); + }); +} + +#[test] +fn published_artifact_upsert_round_trips_and_updates_by_id() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => return, + }; + + sqlx::test_block_on(async { + let pool = PgPool::connect(&database_url) + .await + .expect("connect to DATABASE_URL"); + run_migrations(&pool).await.expect("run migrations"); + let store = PostgresEventStore::new(pool.clone()); + let tenant = format!("tenant-published-artifact-{}", uuid::Uuid::new_v4()); + + let artifact = crate::PostgresPublishedArtifactUpsert { + id: "part-test", + tenant: &tenant, + source_file_id: "file-a", + source_file_version_id: "", + content_hash: "sha256:abc", + label: "latest", + mime_type: "text/markdown", + byte_length: 42, + public_storage_key: "published/file-a.md", + public_url: "https://assets.example/published/file-a.md", + owner_ref_type: "Doc", + owner_ref_id: "doc-a", + status: "published", + }; + + let row = store + .upsert_published_artifact(&artifact) + .await + .expect("upsert published artifact"); + assert_eq!(row.tenant, tenant); + assert_eq!(row.id, "part-test"); + assert_eq!(row.public_url, "https://assets.example/published/file-a.md"); + + let updated = crate::PostgresPublishedArtifactUpsert { + public_url: "https://assets.example/published/file-a-v2.md", + public_storage_key: "published/file-a-v2.md", + byte_length: 43, + ..artifact + }; + store + .upsert_published_artifact(&updated) + .await + .expect("update published artifact"); + + let loaded = store + .load_published_artifact(&tenant, "part-test") + .await + .expect("load published artifact") + .expect("artifact should load after upsert"); + + assert_eq!( + loaded.public_url, + "https://assets.example/published/file-a-v2.md" + ); + assert_eq!(loaded.public_storage_key, "published/file-a-v2.md"); + assert_eq!(loaded.byte_length, 43); + + crate::dbm::postgres_query!("DELETE FROM published_artifacts WHERE tenant = $1") + .bind(&tenant) + .execute(&pool) + .await + .expect("delete test published artifact rows"); + }); +} diff --git a/crates/temper-store-postgres/src/store_projection_test.rs b/crates/temper-store-postgres/src/store_projection_test.rs index 22f3e2df1..a2b3a3833 100644 --- a/crates/temper-store-postgres/src/store_projection_test.rs +++ b/crates/temper-store-postgres/src/store_projection_test.rs @@ -1,7 +1,10 @@ use super::*; use crate::migration::run_migrations; use sqlx::PgPool; -use temper_runtime::persistence::{EntityKeyRow, EventStore}; +use temper_runtime::persistence::{ + EntityKeyRow, EventStore, IndexReconciliation, KeyIndexBackfillFence, PersistenceAppend, + PersistenceBatchIdempotency, SnapshotBackfillFence, +}; fn test_envelope(event_type: &str, payload: serde_json::Value) -> PersistenceEnvelope { PersistenceEnvelope { @@ -89,6 +92,257 @@ fn entity_key_index_present_absent_and_atomic_reject() { Some("doc-a".to_string()), ); + // Exact empty reconciliation co-commits the tombstone and releases A's key. + store + .append_with_keys( + &pid_a, + 1, + &[test_envelope("Deleted", serde_json::json!({}))], + &[], + ) + .await + .unwrap(); + assert_eq!( + store + .lookup_by_key(&tenant, "Doc", "path", &key.key_hash) + .await + .unwrap(), + None, + ); + + // B and C concurrently race to reclaim the vacant key. PostgreSQL's real + // transaction path must commit exactly one and leave the loser's journal + // unchanged. + let pid_c = format!("{tenant}:Doc:doc-c"); + let b_events = [test_envelope("Create", serde_json::json!({}))]; + let c_events = [test_envelope("Create", serde_json::json!({}))]; + let (b_result, c_result) = futures::join!( + store.append_with_keys(&pid_b, 0, &b_events, std::slice::from_ref(&key)), + store.append_with_keys(&pid_c, 0, &c_events, std::slice::from_ref(&key)) + ); + assert_ne!(b_result.is_ok(), c_result.is_ok()); + let (winner_id, winner_pid, loser_pid) = if b_result.is_ok() { + ("doc-b", &pid_b, &pid_c) + } else { + ("doc-c", &pid_c, &pid_b) + }; + assert_eq!( + store + .lookup_by_key(&tenant, "Doc", "path", &key.key_hash) + .await + .unwrap(), + Some(winner_id.to_string()), + ); + assert!(store.read_events(loser_pid, 0).await.unwrap().is_empty()); + + // Interleaving fence: a repair derived from sequence 1 cannot overwrite the + // winner's live sequence-2 rename. + let renamed_key = EntityKeyRow { + key_name: "path".to_string(), + key_hash: format!("renamed-{}", uuid::Uuid::new_v4()), + }; + store + .append_with_keys( + winner_pid, + 1, + &[test_envelope("Rekey", serde_json::json!({}))], + std::slice::from_ref(&renamed_key), + ) + .await + .unwrap(); + let repair_signature = "v3:path"; + let repair_revision = store + .begin_key_index_backfill(&tenant, "Doc", repair_signature) + .await + .unwrap(); + let stale = store + .backfill_entity_keys( + &tenant, + "Doc", + winner_id, + 1, + KeyIndexBackfillFence { + key_set_signature: repair_signature, + contract_revision: repair_revision, + expected_journal_sequence: 1, + expected_entity_live: true, + expected_snapshot: None, + }, + std::slice::from_ref(&key), + ) + .await; + assert!(matches!( + stale, + Err(PersistenceError::JournalBoundaryChanged { + expected: 1, + actual: 2 + }) + )); + assert_eq!( + store + .lookup_by_key(&tenant, "Doc", "path", &key.key_hash) + .await + .unwrap(), + None + ); + assert_eq!( + store + .lookup_by_key(&tenant, "Doc", "path", &renamed_key.key_hash) + .await + .unwrap(), + Some(winner_id.to_string()) + ); + + // Composite batches obey the same exact-key contract. Deleting the + // current owner and creating its replacement in one batch transfers the + // key atomically, independent of append order. + let transfer_pid = format!("{tenant}:Doc:doc-transfer"); + store + .append_batch(&[ + PersistenceAppend { + persistence_id: transfer_pid.clone(), + expected_sequence: 0, + events: vec![test_envelope("Create", serde_json::json!({}))], + key_rows: vec![renamed_key.clone()], + reconcile_keys: true, + key_set_signature: Some("v3:path".to_string()), + snapshot_source: Default::default(), + batch_idempotency: None, + }, + PersistenceAppend { + persistence_id: winner_pid.clone(), + expected_sequence: 2, + events: vec![test_envelope("Deleted", serde_json::json!({}))], + key_rows: Vec::new(), + reconcile_keys: true, + key_set_signature: Some("v3:path".to_string()), + snapshot_source: Default::default(), + batch_idempotency: None, + }, + ]) + .await + .expect("batch delete and reclaim must commit atomically"); + assert_eq!( + store + .lookup_by_key(&tenant, "Doc", "path", &renamed_key.key_hash) + .await + .unwrap(), + Some("doc-transfer".to_string()) + ); + + // A conflict in a later stream rolls back every journal and key row in + // the transaction, including an otherwise-valid earlier claim. + let unrelated_key = EntityKeyRow { + key_name: "path".to_string(), + key_hash: format!("unrelated-{}", uuid::Uuid::new_v4()), + }; + let unrelated_pid = format!("{tenant}:Doc:doc-unrelated"); + let conflict_pid = format!("{tenant}:Doc:doc-conflict"); + let rejected = store + .append_batch(&[ + PersistenceAppend { + persistence_id: unrelated_pid.clone(), + expected_sequence: 0, + events: vec![test_envelope("Create", serde_json::json!({}))], + key_rows: vec![unrelated_key.clone()], + reconcile_keys: true, + key_set_signature: Some("v3:path".to_string()), + snapshot_source: Default::default(), + batch_idempotency: None, + }, + PersistenceAppend { + persistence_id: conflict_pid.clone(), + expected_sequence: 0, + events: vec![test_envelope("Create", serde_json::json!({}))], + key_rows: vec![renamed_key.clone()], + reconcile_keys: true, + key_set_signature: Some("v3:path".to_string()), + snapshot_source: Default::default(), + batch_idempotency: None, + }, + ]) + .await; + assert!( + rejected.is_err(), + "occupied final key must reject the batch" + ); + assert!( + store + .read_events(&unrelated_pid, 0) + .await + .unwrap() + .is_empty() + ); + assert!( + store + .read_events(&conflict_pid, 0) + .await + .unwrap() + .is_empty() + ); + assert_eq!( + store + .lookup_by_key(&tenant, "Doc", "path", &unrelated_key.key_hash) + .await + .unwrap(), + None, + "rejected batch must not publish earlier key rows" + ); + assert_eq!( + store + .lookup_by_key(&tenant, "Doc", "path", &renamed_key.key_hash) + .await + .unwrap(), + Some("doc-transfer".to_string()), + "rejected batch must preserve the committed owner" + ); + + // A durable batch claim is checked before stale sequence fences, and + // binds the idempotency key to the complete intended write. + let idempotent_pid = format!("{tenant}:Repository:repo-idempotent"); + let mut idempotent_append = PersistenceAppend { + persistence_id: idempotent_pid.clone(), + expected_sequence: 0, + events: vec![test_envelope( + "CompositeEvent", + serde_json::json!({"push": 1}), + )], + key_rows: Vec::new(), + reconcile_keys: false, + key_set_signature: None, + snapshot_source: Default::default(), + batch_idempotency: Some(PersistenceBatchIdempotency { + persistence_id: idempotent_pid.clone(), + idempotency_key: "push-1".to_string(), + intent_hash: "intent-a".to_string(), + }), + }; + store + .append_batch(std::slice::from_ref(&idempotent_append)) + .await + .expect("first content-bound batch"); + let retry = store + .append_batch(std::slice::from_ref(&idempotent_append)) + .await + .expect("stale exact retry must be a no-op"); + assert!(retry[0].batch_already_applied); + assert_eq!( + store.read_events(&idempotent_pid, 0).await.unwrap().len(), + 1 + ); + idempotent_append + .batch_idempotency + .as_mut() + .expect("claim") + .intent_hash = "intent-b".to_string(); + let mismatch = store.append_batch(&[idempotent_append]).await; + assert!( + mismatch + .expect_err("same key with different work must fail") + .to_string() + .contains("different intent") + ); + // Clean up this test tenant's rows. let _ = crate::dbm::postgres_query!("DELETE FROM entity_key_index WHERE tenant = $1") .bind(&tenant) @@ -98,13 +352,18 @@ fn entity_key_index_present_absent_and_atomic_reject() { .bind(&tenant) .execute(&pool) .await; + let _ = crate::dbm::postgres_query!( + "DELETE FROM persistence_batch_idempotency WHERE persistence_id = $1" + ) + .bind(&idempotent_pid) + .execute(&pool) + .await; }); } -/// ADR-0153 backfill robustness: the real postgres store honors the backfill -/// primitives the resumable, watermark-gated backfill relies on — -/// `backfill_entity_keys` (no journal event), `keyed_entity_ids_for_type` (resume: -/// which entities are already keyed), and the watermark round-trip +/// ADR-0153/0171 backfill robustness: the real postgres store honors exact, +/// sequence-fenced repair, conflict-before-mutation, index inspection, and the +/// watermark round-trip /// (`mark_key_index_backfilled` / `key_index_backfilled_types`, table from migration /// 0010). Gated on DATABASE_URL; isolated by a unique tenant. #[test] @@ -123,14 +382,32 @@ fn key_index_backfill_and_watermark_methods_round_trip_on_postgres() { key_name: "name_parent".to_string(), key_hash: format!("root-{}", uuid::Uuid::new_v4()), }; + let repair_signature = "name_parent"; + let repair_revision = store + .begin_key_index_backfill(&tenant, "Directory", repair_signature) + .await + .unwrap(); // Backfill (no journal event) keys a pre-existing entity — the root case. store - .backfill_entity_keys(&tenant, "Directory", "dir-root", std::slice::from_ref(&key)) + .backfill_entity_keys( + &tenant, + "Directory", + "dir-root", + 0, + KeyIndexBackfillFence { + key_set_signature: repair_signature, + contract_revision: repair_revision, + expected_journal_sequence: 0, + expected_entity_live: false, + expected_snapshot: None, + }, + std::slice::from_ref(&key), + ) .await .unwrap(); - // Resumability source: the entity now shows as already-keyed for the type. + // Index inspection reports the repaired entity as keyed for the type. assert_eq!( store .keyed_entity_ids_for_type(&tenant, "Directory") @@ -147,6 +424,64 @@ fn key_index_backfill_and_watermark_methods_round_trip_on_postgres() { Some("dir-root".to_string()), ); + // A second historical stream deriving the same current claim makes an exact + // repair incomplete. It must fail (and therefore block the caller's + // watermark), never silently skip the row and report success. + let conflict = store + .backfill_entity_keys( + &tenant, + "Directory", + "dir-conflict", + 0, + KeyIndexBackfillFence { + key_set_signature: repair_signature, + contract_revision: repair_revision, + expected_journal_sequence: 0, + expected_entity_live: false, + expected_snapshot: None, + }, + std::slice::from_ref(&key), + ) + .await; + assert!(matches!( + conflict, + Err(temper_runtime::persistence::PersistenceError::Storage(_)) + )); + assert_eq!( + store + .lookup_by_key(&tenant, "Directory", "name_parent", &key.key_hash) + .await + .unwrap(), + Some("dir-root".to_string()), + "failed repair preserves the established owner" + ); + + // Empty exact backfill is a repair operation, not a no-op. + store + .backfill_entity_keys( + &tenant, + "Directory", + "dir-root", + 0, + KeyIndexBackfillFence { + key_set_signature: repair_signature, + contract_revision: repair_revision, + expected_journal_sequence: 0, + expected_entity_live: false, + expected_snapshot: None, + }, + &[], + ) + .await + .unwrap(); + assert_eq!( + store + .lookup_by_key(&tenant, "Directory", "name_parent", &key.key_hash) + .await + .unwrap(), + None, + ); + // Watermark round-trip: not set until marked, then present for that type only. assert!( store @@ -183,6 +518,149 @@ fn key_index_backfill_and_watermark_methods_round_trip_on_postgres() { vec![("Directory".to_string(), "name_parent,ws_path".to_string())], ); + // Establishing a new target happens before replay and withholds the old + // watermark. A live write under a different signature advances the contract, + // so the stale repair cannot publish even though the write matched the + // contract that existed before repair began. + let target_signature = "v3:directory-target"; + let stable_pid = format!("{tenant}:Directory:dir-stable-race"); + let current_contract_row = EntityKeyRow { + key_name: "current_path".to_string(), + key_hash: format!("current-{}", uuid::Uuid::new_v4()), + }; + store + .append_with_index_rows( + &stable_pid, + 0, + &[test_envelope("Create", serde_json::json!({}))], + std::slice::from_ref(¤t_contract_row), + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(target_signature.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .unwrap(); + let target_revision = store + .begin_key_index_backfill(&tenant, "Directory", target_signature) + .await + .unwrap(); + assert!( + store + .key_index_backfilled_types(&tenant) + .await + .unwrap() + .is_empty() + ); + let live_pid = format!("{tenant}:Directory:dir-live-race"); + store + .append_with_index_rows( + &live_pid, + 0, + &[test_envelope("Create", serde_json::json!({}))], + &[], + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some("v3:directory-old".to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .unwrap(); + let obsolete_row = EntityKeyRow { + key_name: "legacy_path".to_string(), + key_hash: format!("legacy-{}", uuid::Uuid::new_v4()), + }; + let stale_repair = store + .backfill_entity_keys( + &tenant, + "Directory", + "dir-stable-race", + 1, + KeyIndexBackfillFence { + key_set_signature: target_signature, + contract_revision: target_revision, + expected_journal_sequence: 1, + expected_entity_live: true, + expected_snapshot: None, + }, + std::slice::from_ref(&obsolete_row), + ) + .await; + assert!(matches!( + stale_repair, + Err(PersistenceError::KeyContractChanged { + expected_revision, + actual_revision, + .. + }) if expected_revision == target_revision && actual_revision > target_revision + )); + assert_eq!( + store + .lookup_by_key( + &tenant, + "Directory", + ¤t_contract_row.key_name, + ¤t_contract_row.key_hash, + ) + .await + .unwrap(), + Some("dir-stable-race".to_string()), + "cross-stream contract fence must preserve current rows" + ); + assert_eq!( + store + .lookup_by_key( + &tenant, + "Directory", + &obsolete_row.key_name, + &obsolete_row.key_hash, + ) + .await + .unwrap(), + None, + "cross-stream contract fence must reject obsolete rows" + ); + assert!( + !store + .mark_key_index_backfilled_if_revision( + &tenant, + "Directory", + target_signature, + target_revision, + ) + .await + .unwrap(), + "a mixed-contract live write must fence stale watermark publication" + ); + assert!( + store + .key_index_backfilled_types(&tenant) + .await + .unwrap() + .is_empty() + ); + let fresh_revision = store + .begin_key_index_backfill(&tenant, "Directory", target_signature) + .await + .unwrap(); + assert!( + store + .mark_key_index_backfilled_if_revision( + &tenant, + "Directory", + target_signature, + fresh_revision, + ) + .await + .unwrap() + ); + let _ = crate::dbm::postgres_query!("DELETE FROM entity_key_index WHERE tenant = $1") .bind(&tenant) .execute(&pool) @@ -193,11 +671,16 @@ fn key_index_backfill_and_watermark_methods_round_trip_on_postgres() { .bind(&tenant) .execute(&pool) .await; + let _ = + crate::dbm::postgres_query!("DELETE FROM key_index_contract_state WHERE tenant = $1") + .bind(&tenant) + .execute(&pool) + .await; }); } #[test] -fn list_entity_ids_by_type_unions_catalog_field_index_and_events() { +fn entity_listing_and_key_repair_union_every_durable_source() { let database_url = match std::env::var("DATABASE_URL") { Ok(url) => url, Err(_) => return, @@ -254,14 +737,62 @@ fn list_entity_ids_by_type_unions_catalog_field_index_and_events() { ) .await .unwrap(); + store + .append( + &format!("{tenant}:{entity_type}:dl-explicit-live-legacy-name"), + 0, + &[test_envelope( + "Deleted", + serde_json::json!({ + "action": "Deleted", + "from_status": "Published", + "to_status": "Published", + }), + )], + ) + .await + .unwrap(); store .append( &format!("{tenant}:{entity_type}:dl-deleted"), 0, - &[test_envelope("Deleted", serde_json::json!({}))], + &[test_envelope( + "Delete", + serde_json::json!({ + "action": "Delete", + "from_status": "Published", + "to_status": "Deleted", + }), + )], + ) + .await + .unwrap(); + store + .save_snapshot( + &format!("{tenant}:{entity_type}:dl-deleted"), + 3, + b"newer-stale-live-snapshot", ) .await .unwrap(); + store + .save_snapshot( + &format!("{tenant}:{entity_type}:dl-snapshot"), + 4, + b"snapshot-only", + ) + .await + .unwrap(); + crate::dbm::postgres_query!( + "INSERT INTO entity_key_index \ + (tenant, entity_type, entity_id, key_name, key_hash, sequence_nr) \ + VALUES ($1, $2, 'dl-key-only', 'path', 'orphan', 0)", + ) + .bind(&tenant) + .bind(entity_type) + .execute(&pool) + .await + .unwrap(); let ids = store .list_entity_ids_by_type(&tenant, entity_type) @@ -273,10 +804,65 @@ fn list_entity_ids_by_type_unions_catalog_field_index_and_events() { vec![ "dl-catalog".to_string(), "dl-event".to_string(), + "dl-explicit-live-legacy-name".to_string(), "dl-index".to_string(), + "dl-snapshot".to_string(), ] ); + let repair_signature = "v3:name"; + let repair_revision = store + .begin_key_index_backfill(&tenant, entity_type, repair_signature) + .await + .unwrap(); + let stale_live_repair = store + .backfill_entity_keys( + &tenant, + entity_type, + "dl-deleted", + 3, + KeyIndexBackfillFence { + key_set_signature: repair_signature, + contract_revision: repair_revision, + expected_journal_sequence: 1, + expected_entity_live: true, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: 3, + state: b"newer-stale-live-snapshot", + }), + }, + &[], + ) + .await; + assert!( + matches!( + &stale_live_repair, + Err(PersistenceError::EntityLivenessChanged { + expected_live: true, + actual_live: false, + }) + ), + "unexpected stale-live repair result: {stale_live_repair:?}" + ); + + let repair_ids = store + .list_entity_ids_for_key_reconciliation(&tenant, entity_type) + .await + .unwrap(); + assert_eq!( + repair_ids, + vec![ + "dl-catalog".to_string(), + "dl-deleted".to_string(), + "dl-event".to_string(), + "dl-explicit-live-legacy-name".to_string(), + "dl-index".to_string(), + "dl-key-only".to_string(), + "dl-snapshot".to_string(), + ], + "repair coverage must include live, deleted, derived-only, and migration-only rows" + ); + crate::dbm::postgres_query!("DELETE FROM entity_field_index WHERE tenant = $1") .bind(&tenant) .execute(&pool) @@ -287,11 +873,26 @@ fn list_entity_ids_by_type_unions_catalog_field_index_and_events() { .execute(&pool) .await .unwrap(); + crate::dbm::postgres_query!("DELETE FROM entity_key_index WHERE tenant = $1") + .bind(&tenant) + .execute(&pool) + .await + .unwrap(); + crate::dbm::postgres_query!("DELETE FROM snapshots WHERE tenant = $1") + .bind(&tenant) + .execute(&pool) + .await + .unwrap(); crate::dbm::postgres_query!("DELETE FROM events WHERE tenant = $1") .bind(&tenant) .execute(&pool) .await .unwrap(); + crate::dbm::postgres_query!("DELETE FROM event_segments WHERE tenant = $1") + .bind(&tenant) + .execute(&pool) + .await + .unwrap(); }); } @@ -707,6 +1308,16 @@ fn native_data_only_create_inserts_event_catalog_and_index_atomically() { let mut envelope = test_envelope("Created", fields.clone()); envelope.sequence_nr = 1; + let key_signature = "v3|session_entry[SessionId,EntryId]"; + store + .mark_key_index_backfilled(&tenant, entity_type, key_signature) + .await + .expect("seed complete declared-key coverage"); + let revision_before = store + .key_index_reconciliation_revision(&tenant, entity_type) + .await + .expect("read pre-create key revision"); + let sequence_nr = store .create_data_only_entity_native( &tenant, @@ -720,6 +1331,37 @@ fn native_data_only_create_inserts_event_catalog_and_index_atomically() { .unwrap(); assert_eq!(sequence_nr, 1); + assert!( + !store + .key_index_backfilled_types(&tenant) + .await + .expect("read invalidated key coverage") + .iter() + .any(|(covered_type, _)| covered_type == entity_type), + "a legacy native create without exact key rows must invalidate authoritative keyed absence" + ); + assert!( + store + .key_index_reconciliation_revision(&tenant, entity_type) + .await + .expect("read post-create key revision") + > revision_before, + "coverage invalidation must advance the contract revision" + ); + let dirty_projection_count: i64 = crate::dbm::postgres_query_scalar!( + "SELECT COUNT(*)::bigint FROM query_projection_dirty \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(&tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!( + dirty_projection_count, 0, + "the atomic journal+catalog fast path must close its trigger-created repair marker" + ); let event_count: i64 = crate::dbm::postgres_query_scalar!( "SELECT COUNT(*)::bigint FROM events \ WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", @@ -1144,3 +1786,169 @@ fn upsert_query_projection_removes_index_row_when_value_becomes_too_long() { .unwrap(); }); } + +#[test] +fn journal_generation_replaces_a_numerically_newer_migration_projection() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => return, + }; + + sqlx::test_block_on(async { + let pool = PgPool::connect(&database_url).await.unwrap(); + run_migrations(&pool).await.unwrap(); + let store = PostgresEventStore::new(pool.clone()); + let tenant = format!("tenant-journal-projection-{}", uuid::Uuid::new_v4()); + let entity_type = "Order"; + let entity_id = "ord-materialized"; + let persistence_id = format!("{tenant}:{entity_type}:{entity_id}"); + + store + .upsert_query_projection( + &tenant, + entity_type, + entity_id, + "Draft", + &serde_json::json!({"Title": "migration"}), + 5, + ) + .await + .unwrap(); + store + .append( + &persistence_id, + 0, + &[ + test_envelope( + "Temper.Internal.StateMaterialization.v1", + serde_json::json!({}), + ), + test_envelope("Touch", serde_json::json!({})), + ], + ) + .await + .unwrap(); + store + .upsert_query_projection( + &tenant, + entity_type, + entity_id, + "Ready", + &serde_json::json!({"Title": "journal"}), + 2, + ) + .await + .unwrap(); + + let row: (i64, serde_json::Value) = crate::dbm::postgres_query_as!( + "SELECT sequence_nr, fields FROM entity_catalog \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(&tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(row.0, 2); + assert_eq!(row.1["Title"], "journal"); + + let journal_indexed: i64 = crate::dbm::postgres_query_scalar!( + "SELECT COUNT(*)::bigint FROM entity_field_index \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ + AND field_name = 'Title' AND field_value = 'journal'", + ) + .bind(&tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(journal_indexed, 1); + let migration_indexed: i64 = crate::dbm::postgres_query_scalar!( + "SELECT COUNT(*)::bigint FROM entity_field_index \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ + AND field_name = 'Title' AND field_value = 'migration'", + ) + .bind(&tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(migration_indexed, 0); + + for (delayed_sequence, title) in [(5, "delayed-migration"), (1, "stale-journal")] { + store + .upsert_query_projection( + &tenant, + entity_type, + entity_id, + "Delayed", + &serde_json::json!({"Title": title}), + delayed_sequence, + ) + .await + .unwrap(); + } + let current: (i64, serde_json::Value, String) = crate::dbm::postgres_query_as!( + "SELECT sequence_nr, fields, status FROM entity_catalog \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(&tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(current.0, 2); + assert_eq!(current.1["Title"], "journal"); + assert_eq!(current.2, "Ready"); + let journal_still_indexed: i64 = crate::dbm::postgres_query_scalar!( + "SELECT COUNT(*)::bigint FROM entity_field_index \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ + AND field_name = 'Title' AND field_value = 'journal' AND status = 'Ready'", + ) + .bind(&tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(journal_still_indexed, 1); + let rejected_indexed: i64 = crate::dbm::postgres_query_scalar!( + "SELECT COUNT(*)::bigint FROM entity_field_index \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ + AND field_name = 'Title' AND field_value IN ('delayed-migration', 'stale-journal')", + ) + .bind(&tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(rejected_indexed, 0); + + crate::dbm::postgres_query!("DELETE FROM event_segments WHERE tenant = $1") + .bind(&tenant) + .execute(&pool) + .await + .unwrap(); + + crate::dbm::postgres_query!("DELETE FROM entity_field_index WHERE tenant = $1") + .bind(&tenant) + .execute(&pool) + .await + .unwrap(); + crate::dbm::postgres_query!("DELETE FROM entity_catalog WHERE tenant = $1") + .bind(&tenant) + .execute(&pool) + .await + .unwrap(); + crate::dbm::postgres_query!("DELETE FROM events WHERE tenant = $1") + .bind(&tenant) + .execute(&pool) + .await + .unwrap(); + }); +} diff --git a/crates/temper-store-postgres/tests/key_repair_liveness_race.rs b/crates/temper-store-postgres/tests/key_repair_liveness_race.rs new file mode 100644 index 000000000..01b29d993 --- /dev/null +++ b/crates/temper-store-postgres/tests/key_repair_liveness_race.rs @@ -0,0 +1,132 @@ +//! Regression for an orphan repair candidate becoming live after enumeration. + +use temper_runtime::persistence::{ + EntityKeyRow, EventMetadata, EventStore, IndexReconciliation, KeyIndexBackfillFence, + PersistenceEnvelope, +}; +use temper_runtime::scheduler::{sim_now, sim_uuid}; +use temper_store_postgres::PostgresEventStore; + +#[test] +fn stale_orphan_classification_cannot_delete_a_concurrent_live_claim() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => return, + }; + sqlx::test_block_on(async { + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect to Postgres"); + temper_store_postgres::migration::run_migrations(&pool) + .await + .expect("run Postgres migrations"); + let store = PostgresEventStore::new(pool.clone()); + let tenant = format!("arn238-key-race-{}", sim_uuid()); + let entity_type = "Doc"; + let entity_id = "orphan-becomes-live"; + let key = EntityKeyRow { + key_name: "path".to_string(), + key_hash: format!("path-{}", sim_uuid()), + }; + let key_set_signature = "v3:path"; + + sqlx::query( + "INSERT INTO entity_key_index \ + (tenant, entity_type, key_name, key_hash, entity_id, sequence_nr) \ + VALUES ($1, $2, $3, $4, $5, 0)", + ) + .bind(&tenant) + .bind(entity_type) + .bind(&key.key_name) + .bind(&key.key_hash) + .bind(entity_id) + .execute(&pool) + .await + .expect("seed key-only orphan"); + + let repair_revision = store + .begin_key_index_backfill(&tenant, entity_type, key_set_signature) + .await + .expect("begin repair contract"); + assert_eq!( + store + .list_entity_ids_for_key_reconciliation(&tenant, entity_type) + .await + .expect("enumerate repair candidates"), + vec![entity_id.to_string()] + ); + assert!( + store + .list_entity_ids_by_type(&tenant, entity_type) + .await + .expect("classify live entities") + .is_empty(), + "precondition: the key-only repair candidate is initially not live" + ); + + let persistence_id = format!("{tenant}:{entity_type}:{entity_id}"); + let timestamp = sim_now(); + store + .append_with_index_rows( + &persistence_id, + 0, + &[PersistenceEnvelope { + sequence_nr: 1, + event_type: "Create".to_string(), + payload: serde_json::json!({}), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: persistence_id.clone(), + }, + }], + std::slice::from_ref(&key), + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(key_set_signature.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .expect("same-contract create after repair enumeration"); + + let stale_repair = store + .backfill_entity_keys( + &tenant, + entity_type, + entity_id, + 1, + KeyIndexBackfillFence { + key_set_signature, + contract_revision: repair_revision, + expected_journal_sequence: 1, + expected_entity_live: false, + expected_snapshot: None, + }, + &[], + ) + .await; + + assert!(matches!( + stale_repair, + Err( + temper_runtime::persistence::PersistenceError::EntityLivenessChanged { + expected_live: false, + actual_live: true, + } + ) + )); + assert_eq!( + store + .lookup_by_key(&tenant, entity_type, &key.key_name, &key.key_hash) + .await + .expect("lookup concurrent live claim"), + Some(entity_id.to_string()), + "a stale repair must preserve the co-committed live key" + ); + }); +} diff --git a/crates/temper-store-postgres/tests/key_repair_writer_fence.rs b/crates/temper-store-postgres/tests/key_repair_writer_fence.rs new file mode 100644 index 000000000..14fdaa189 --- /dev/null +++ b/crates/temper-store-postgres/tests/key_repair_writer_fence.rs @@ -0,0 +1,980 @@ +//! Regression for derived durable writers racing exact key-index repair. + +use futures::future::{Either, select}; +use temper_runtime::persistence::{ + EntityKeyRow, EventMetadata, EventStore, IndexReconciliation, KeyIndexBackfillFence, + PersistenceEnvelope, PersistenceError, SnapshotBackfillFence, SnapshotSourceFence, + encode_activated_key_contract, +}; +use temper_runtime::scheduler::{sim_now, sim_uuid}; +use temper_store_postgres::PostgresEventStore; + +type SegmentRow = (i64, i64, Option, Option, i64, bool, String); + +async fn hold_stream_fence<'a>( + pool: &'a sqlx::PgPool, + persistence_id: &str, +) -> sqlx::Transaction<'a, sqlx::Postgres> { + let mut transaction = pool.begin().await.expect("begin stream-fence holder"); + sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))") + .bind(persistence_id) + .execute(&mut *transaction) + .await + .expect("hold exact-repair stream fence"); + transaction +} + +#[test] +fn lower_authoritative_snapshot_outranks_stale_catalog_during_key_backfill() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => return, + }; + sqlx::test_block_on(async { + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect to Postgres"); + temper_store_postgres::migration::run_migrations(&pool) + .await + .expect("run Postgres migrations"); + let store = PostgresEventStore::new(pool); + let tenant = format!("arn238-snapshot-catalog-precedence-{}", sim_uuid()); + let entity_type = "Doc"; + let entity_id = "snapshot-authority"; + let persistence_id = format!("{tenant}:{entity_type}:{entity_id}"); + let signature = "v4:path"; + let snapshot = br#"{"fields":{"Path":"/snapshot"}}"#; + let key = EntityKeyRow { + key_name: "path".to_string(), + key_hash: format!("snapshot-owner-{}", sim_uuid()), + }; + + store + .save_snapshot(&persistence_id, 5, snapshot) + .await + .expect("seed authoritative snapshot generation"); + store + .upsert_query_projection( + &tenant, + entity_type, + entity_id, + "Live", + &serde_json::json!({"Path": "/stale-catalog"}), + 10, + ) + .await + .expect("seed newer compatibility catalog projection"); + let revision = store + .begin_key_index_backfill(&tenant, entity_type, signature) + .await + .expect("begin exact key repair"); + + store + .backfill_entity_keys( + &tenant, + entity_type, + entity_id, + 5, + KeyIndexBackfillFence { + key_set_signature: signature, + contract_revision: revision, + expected_journal_sequence: 0, + expected_entity_live: true, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: 5, + state: snapshot, + }), + }, + std::slice::from_ref(&key), + ) + .await + .expect("snapshot generation must fence key repair ahead of stale catalog sequence"); + assert!( + store + .mark_key_index_backfilled_if_revision(&tenant, entity_type, signature, revision,) + .await + .expect("publish snapshot-derived coverage"), + "stable snapshot-derived ownership must publish coverage" + ); + assert_eq!( + store + .lookup_by_key(&tenant, entity_type, "path", &key.key_hash) + .await + .expect("lookup snapshot-derived key owner"), + Some(entity_id.to_string()) + ); + assert_eq!( + store + .key_index_backfilled_types(&tenant) + .await + .expect("load published coverage"), + vec![(entity_type.to_string(), signature.to_string())] + ); + }); +} + +#[test] +fn stale_writer_epoch_cannot_resurrect_a_claim_after_a_none_a_activation() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => return, + }; + sqlx::test_block_on(async { + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect to Postgres"); + temper_store_postgres::migration::run_migrations(&pool) + .await + .expect("run Postgres migrations"); + let store = PostgresEventStore::new(pool); + let tenant = format!("arn238-activation-epoch-{}", sim_uuid()); + let entity_type = "Doc"; + let entity_id = "delayed-writer"; + let persistence_id = format!("{tenant}:{entity_type}:{entity_id}"); + let signature_a = "v3|4:path[4:Path]"; + let signature_none = "v3"; + let row = EntityKeyRow { + key_name: "path".to_string(), + key_hash: "released".to_string(), + }; + let envelope = |event_type: &str| PersistenceEnvelope { + sequence_nr: 0, + event_type: event_type.to_string(), + payload: serde_json::json!({}), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: sim_now(), + actor_id: persistence_id.clone(), + }, + }; + + let old_epoch = store + .activate_key_index_contract(&tenant, entity_type, signature_a, false) + .await + .expect("activate original A contract"); + store + .mark_key_index_backfilled(&tenant, entity_type, signature_a) + .await + .expect("publish original A readiness"); + let old_contract = encode_activated_key_contract(signature_a, old_epoch); + store + .append_with_index_rows( + &persistence_id, + 0, + &[envelope("Created")], + std::slice::from_ref(&row), + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(old_contract.clone()), + ..IndexReconciliation::default() + }, + ) + .await + .expect("seed A owner"); + store + .activate_key_index_contract(&tenant, entity_type, signature_none, true) + .await + .expect("activate empty contract and purge rows"); + let current_epoch = store + .activate_key_index_contract(&tenant, entity_type, signature_a, false) + .await + .expect("reactivate A at a new epoch"); + + let error = store + .append_with_index_rows( + &persistence_id, + 1, + &[envelope("DelayedOldWriter")], + std::slice::from_ref(&row), + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(old_contract), + ..IndexReconciliation::default() + }, + ) + .await + .expect_err("old A epoch must be rejected after A is reactivated"); + assert!(matches!( + error, + PersistenceError::KeyContractActivationStale { + activated_epoch, + attempted_epoch: Some(attempted_epoch), + } if activated_epoch == current_epoch && attempted_epoch == old_epoch + )); + assert_eq!( + store + .read_events(&persistence_id, 0) + .await + .expect("read journal") + .len(), + 1, + "rejected writer must not advance the journal" + ); + assert_eq!( + store + .lookup_by_key(&tenant, entity_type, "path", &row.key_hash) + .await + .expect("read key owner"), + None, + "rejected writer must not resurrect the purged row" + ); + }); +} + +#[test] +fn unreconciled_writer_between_entity_repair_and_publication_closes_readiness() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => return, + }; + sqlx::test_block_on(async { + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect to Postgres"); + temper_store_postgres::migration::run_migrations(&pool) + .await + .expect("run Postgres migrations"); + let store = PostgresEventStore::new(pool); + let tenant = format!("arn238-unreconciled-race-{}", sim_uuid()); + let entity_type = "Doc"; + let entity_id = "writer-during-repair"; + let persistence_id = format!("{tenant}:{entity_type}:{entity_id}"); + let key_set = "v3|4:path[4:Path]"; + let key = EntityKeyRow { + key_name: "path".to_string(), + key_hash: "writer-during-repair".to_string(), + }; + let envelope = |event_type: &str| PersistenceEnvelope { + sequence_nr: 0, + event_type: event_type.to_string(), + payload: serde_json::json!({}), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: sim_now(), + actor_id: persistence_id.clone(), + }, + }; + + store + .append(&persistence_id, 0, &[envelope("Created")]) + .await + .expect("seed pre-contract journal"); + let repair_revision = store + .begin_key_index_backfill(&tenant, entity_type, key_set) + .await + .expect("begin key repair"); + store + .backfill_entity_keys( + &tenant, + entity_type, + entity_id, + 1, + KeyIndexBackfillFence { + key_set_signature: key_set, + contract_revision: repair_revision, + expected_journal_sequence: 1, + expected_entity_live: true, + expected_snapshot: None, + }, + std::slice::from_ref(&key), + ) + .await + .expect("repair entity before racing writer"); + + store + .append(&persistence_id, 1, &[envelope("UnreconciledWriter")]) + .await + .expect("commit unreconciled writer in publication window"); + + assert!( + !store + .mark_key_index_backfilled_if_revision( + &tenant, + entity_type, + key_set, + repair_revision, + ) + .await + .expect("attempt stale readiness publication"), + "the writer must make the captured repair revision lose its CAS" + ); + assert!( + store + .key_index_backfilled_types(&tenant) + .await + .expect("read closed readiness") + .is_empty(), + "a partially repaired row set must never become authoritative" + ); + assert!( + store + .key_index_reconciliation_revision(&tenant, entity_type) + .await + .expect("read post-writer revision") + > repair_revision + ); + }); +} + +#[test] +fn snapshot_writer_waits_for_exact_repair_stream_fence() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => return, + }; + sqlx::test_block_on(async { + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect to Postgres"); + temper_store_postgres::migration::run_migrations(&pool) + .await + .expect("run Postgres migrations"); + let store = PostgresEventStore::new(pool.clone()); + let tenant = format!("arn238-snapshot-fence-{}", sim_uuid()); + let persistence_id = format!("{tenant}:Doc:snapshot-only"); + let blocker = hold_stream_fence(&pool, &persistence_id).await; + + let write = Box::pin(store.save_snapshot(&persistence_id, 1, b"snapshot-only")); + let deadline = Box::pin(sqlx::query("SELECT pg_sleep(1)").execute(&pool)); + let write = match select(write, deadline).await { + Either::Left((result, _)) => panic!( + "snapshot writer crossed the exact-repair stream fence before release: {result:?}" + ), + Either::Right((deadline, write)) => { + deadline.expect("snapshot fence deadline query"); + write + } + }; + + blocker.commit().await.expect("release stream fence"); + write + .await + .expect("snapshot writer resumes after fence release"); + }); +} + +#[test] +fn stale_snapshot_sources_leave_journal_keys_history_and_segments_unchanged() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => return, + }; + sqlx::test_block_on(async { + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect to Postgres"); + temper_store_postgres::migration::run_migrations(&pool) + .await + .expect("run Postgres migrations"); + let store = PostgresEventStore::new(pool.clone()); + let tenant = format!("arn238-stale-writer-nonmutation-{}", sim_uuid()); + let entity_type = "Doc"; + let entity_id = "stale-source"; + let persistence_id = format!("{tenant}:{entity_type}:{entity_id}"); + let timestamp = sim_now(); + let envelope = |event_type: &str| PersistenceEnvelope { + sequence_nr: 0, + event_type: event_type.to_string(), + payload: serde_json::json!({"event": event_type}), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: persistence_id.clone(), + }, + }; + let original_key = EntityKeyRow { + key_name: "path".to_string(), + key_hash: "original".to_string(), + }; + let rejected_key = EntityKeyRow { + key_name: "path".to_string(), + key_hash: "rejected".to_string(), + }; + let signature = "v4:path"; + store + .append_with_index_rows( + &persistence_id, + 0, + &[envelope("Created")], + std::slice::from_ref(&original_key), + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(signature.to_string()), + vectors: false, + snapshot_source: SnapshotSourceFence::Absent, + }, + ) + .await + .expect("seed journal and key ownership"); + let captured_snapshot = b"captured".to_vec(); + let replacement_snapshot = b"replacement".to_vec(); + store + .save_snapshot(&persistence_id, 1, &captured_snapshot) + .await + .expect("seed captured snapshot"); + store + .save_snapshot(&persistence_id, 1, &replacement_snapshot) + .await + .expect("replace captured snapshot generation"); + + let events_before: Vec<(i64, String, serde_json::Value, i64)> = sqlx::query_as( + "SELECT sequence_nr, event_type, payload, segment_index FROM events \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 ORDER BY sequence_nr", + ) + .bind(&tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_all(&pool) + .await + .expect("capture journal rows"); + let segments_before: Vec = sqlx::query_as( + "SELECT segment_index, start_sequence_nr, end_sequence_nr, snapshot_sequence, \ + event_count, sealed_at IS NOT NULL, xmin::text \ + FROM event_segments WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ + ORDER BY segment_index", + ) + .bind(&tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_all(&pool) + .await + .expect("capture segment rows"); + let history_before: Vec<(i64, Vec, String)> = sqlx::query_as( + "SELECT sequence_nr, state, xmin::text FROM snapshot_history \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 ORDER BY sequence_nr", + ) + .bind(&tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_all(&pool) + .await + .expect("capture snapshot history"); + + for stale_source in [ + SnapshotSourceFence::Exact { + sequence_nr: 1, + state: captured_snapshot.clone(), + }, + SnapshotSourceFence::Absent, + ] { + let error = store + .append_with_index_rows( + &persistence_id, + 1, + &[envelope("Rejected")], + std::slice::from_ref(&rejected_key), + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(signature.to_string()), + vectors: false, + snapshot_source: stale_source, + }, + ) + .await + .expect_err("stale snapshot source must reject append"); + assert!(matches!(error, PersistenceError::SnapshotGenerationChanged)); + } + let checked_error = store + .save_snapshot_if_source( + &persistence_id, + 2, + b"must-not-commit", + &SnapshotSourceFence::Exact { + sequence_nr: 1, + state: captured_snapshot, + }, + None, + ) + .await + .expect_err("stale conditional snapshot save must reject"); + assert!(matches!( + checked_error, + PersistenceError::SnapshotGenerationChanged + )); + + let events_after: Vec<(i64, String, serde_json::Value, i64)> = sqlx::query_as( + "SELECT sequence_nr, event_type, payload, segment_index FROM events \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 ORDER BY sequence_nr", + ) + .bind(&tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_all(&pool) + .await + .expect("reload journal rows"); + let segments_after: Vec = sqlx::query_as( + "SELECT segment_index, start_sequence_nr, end_sequence_nr, snapshot_sequence, \ + event_count, sealed_at IS NOT NULL, xmin::text \ + FROM event_segments WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ + ORDER BY segment_index", + ) + .bind(&tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_all(&pool) + .await + .expect("reload segment rows"); + let history_after: Vec<(i64, Vec, String)> = sqlx::query_as( + "SELECT sequence_nr, state, xmin::text FROM snapshot_history \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 ORDER BY sequence_nr", + ) + .bind(&tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_all(&pool) + .await + .expect("reload snapshot history"); + assert_eq!(events_after, events_before); + assert_eq!(segments_after, segments_before); + assert_eq!(history_after, history_before); + assert_eq!( + store.load_snapshot(&persistence_id).await.unwrap(), + Some((1, replacement_snapshot)) + ); + assert_eq!( + store + .lookup_by_key(&tenant, entity_type, "path", &original_key.key_hash) + .await + .unwrap(), + Some(entity_id.to_string()) + ); + assert_eq!( + store + .lookup_by_key(&tenant, entity_type, "path", &rejected_key.key_hash) + .await + .unwrap(), + None + ); + }); +} + +#[test] +fn equal_snapshot_rewrite_wins_shared_stream_fence_before_stale_append() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => return, + }; + sqlx::test_block_on(async { + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect to Postgres"); + temper_store_postgres::migration::run_migrations(&pool) + .await + .expect("run Postgres migrations"); + let store = PostgresEventStore::new(pool.clone()); + let tenant = format!("arn238-append-snapshot-race-{}", sim_uuid()); + let persistence_id = format!("{tenant}:Doc:shared-stream-fence"); + let event = |event_type: &str| PersistenceEnvelope { + sequence_nr: 0, + event_type: event_type.to_string(), + payload: serde_json::json!({}), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: sim_now(), + actor_id: persistence_id.clone(), + }, + }; + store + .append(&persistence_id, 0, &[event("Created")]) + .await + .expect("seed journal"); + let captured = b"captured".to_vec(); + let replacement = b"replacement".to_vec(); + store + .save_snapshot(&persistence_id, 1, &captured) + .await + .expect("seed captured snapshot"); + let blocker = hold_stream_fence(&pool, &persistence_id).await; + + let rewrite = Box::pin(store.save_snapshot(&persistence_id, 1, &replacement)); + let deadline = Box::pin(sqlx::query("SELECT pg_sleep(0.25)").execute(&pool)); + let rewrite = match select(rewrite, deadline).await { + Either::Left((result, _)) => { + panic!("snapshot rewrite crossed held stream fence: {result:?}") + } + Either::Right((deadline, rewrite)) => { + deadline.expect("snapshot rewrite wait deadline"); + rewrite + } + }; + + let rejected_events = vec![event("MustReject")]; + let append = Box::pin(store.append_with_index_rows( + &persistence_id, + 1, + &rejected_events, + &[], + &[], + IndexReconciliation { + snapshot_source: SnapshotSourceFence::Exact { + sequence_nr: 1, + state: captured, + }, + ..IndexReconciliation::default() + }, + )); + let deadline = Box::pin(sqlx::query("SELECT pg_sleep(0.25)").execute(&pool)); + let append = match select(append, deadline).await { + Either::Left((result, _)) => { + panic!("append crossed held stream fence: {result:?}") + } + Either::Right((deadline, append)) => { + deadline.expect("append wait deadline"); + append + } + }; + + blocker.commit().await.expect("release shared stream fence"); + rewrite + .await + .expect("queued snapshot rewrite commits first"); + let append_error = append.await.expect_err("stale append must reject"); + assert!(matches!( + append_error, + PersistenceError::SnapshotGenerationChanged + )); + assert_eq!( + store.read_events(&persistence_id, 0).await.unwrap().len(), + 1 + ); + assert_eq!( + store.load_snapshot(&persistence_id).await.unwrap(), + Some((1, replacement)) + ); + }); +} + +#[test] +fn catalog_writer_waits_for_exact_repair_stream_fence() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => return, + }; + sqlx::test_block_on(async { + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect to Postgres"); + temper_store_postgres::migration::run_migrations(&pool) + .await + .expect("run Postgres migrations"); + let store = PostgresEventStore::new(pool.clone()); + let tenant = format!("arn238-catalog-fence-{}", sim_uuid()); + let persistence_id = format!("{tenant}:Doc:catalog-only"); + let blocker = hold_stream_fence(&pool, &persistence_id).await; + + let fields = serde_json::json!({"WorkspaceId": "ws", "Path": "/catalog-only"}); + let write = Box::pin(store.upsert_query_projection( + &tenant, + "Doc", + "catalog-only", + "Ready", + &fields, + 1, + )); + let deadline = Box::pin(sqlx::query("SELECT pg_sleep(1)").execute(&pool)); + let write = match select(write, deadline).await { + Either::Left((result, _)) => panic!( + "catalog writer crossed the exact-repair stream fence before release: {result:?}" + ), + Either::Right((deadline, write)) => { + deadline.expect("catalog fence deadline query"); + write + } + }; + + blocker.commit().await.expect("release stream fence"); + write + .await + .expect("catalog writer resumes after fence release"); + }); +} + +#[test] +fn snapshot_only_writer_invalidates_inflight_key_coverage() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => return, + }; + sqlx::test_block_on(async { + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect to Postgres"); + temper_store_postgres::migration::run_migrations(&pool) + .await + .expect("run Postgres migrations"); + let store = PostgresEventStore::new(pool); + let tenant = format!("arn238-snapshot-coverage-{}", sim_uuid()); + let signature = "v4:path"; + let revision = store + .begin_key_index_backfill(&tenant, "Doc", signature) + .await + .expect("begin key repair"); + + store + .save_snapshot(&format!("{tenant}:Doc:snapshot-only"), 1, b"snapshot-only") + .await + .expect("write newly enumerated snapshot owner"); + + assert!( + !store + .mark_key_index_backfilled_if_revision(&tenant, "Doc", signature, revision) + .await + .expect("conditionally publish coverage"), + "a snapshot-only owner created after enumeration must reject publication" + ); + }); +} + +#[test] +fn catalog_only_writer_invalidates_inflight_key_coverage() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => return, + }; + sqlx::test_block_on(async { + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect to Postgres"); + temper_store_postgres::migration::run_migrations(&pool) + .await + .expect("run Postgres migrations"); + let store = PostgresEventStore::new(pool); + let tenant = format!("arn238-catalog-coverage-{}", sim_uuid()); + let signature = "v4:path"; + let revision = store + .begin_key_index_backfill(&tenant, "Doc", signature) + .await + .expect("begin key repair"); + + store + .upsert_query_projection( + &tenant, + "Doc", + "catalog-only", + "Ready", + &serde_json::json!({"WorkspaceId": "ws", "Path": "/catalog-only"}), + 1, + ) + .await + .expect("write newly enumerated catalog owner"); + + assert!( + !store + .mark_key_index_backfilled_if_revision(&tenant, "Doc", signature, revision) + .await + .expect("conditionally publish coverage"), + "a catalog-only owner created after enumeration must reject publication" + ); + }); +} + +#[test] +fn journal_source_fence_rejects_equal_sequence_snapshot_repair() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => return, + }; + sqlx::test_block_on(async { + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect to Postgres"); + temper_store_postgres::migration::run_migrations(&pool) + .await + .expect("run Postgres migrations"); + let store = PostgresEventStore::new(pool); + let tenant = format!("arn238-source-fence-{}", sim_uuid()); + let entity_type = "Doc"; + let entity_id = "source-aba"; + let persistence_id = format!("{tenant}:{entity_type}:{entity_id}"); + let repair_signature = "v4:path"; + let snapshot_key = EntityKeyRow { + key_name: "path".to_string(), + key_hash: format!("snapshot-path-{}", sim_uuid()), + }; + let journal_key = EntityKeyRow { + key_name: "path".to_string(), + key_hash: format!("journal-path-{}", sim_uuid()), + }; + + store + .save_snapshot(&persistence_id, 1, b"snapshot-only") + .await + .expect("seed snapshot-only generation"); + let repair_revision = store + .begin_key_index_backfill(&tenant, entity_type, repair_signature) + .await + .expect("begin snapshot-derived repair"); + + let timestamp = sim_now(); + store + .append_with_index_rows( + &persistence_id, + 0, + &[PersistenceEnvelope { + sequence_nr: 1, + event_type: "Create".to_string(), + payload: serde_json::json!({}), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: persistence_id.clone(), + }, + }], + std::slice::from_ref(&journal_key), + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(repair_signature.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .expect("replace snapshot-only source with equal-sequence journal state"); + + let stale = store + .backfill_entity_keys( + &tenant, + entity_type, + entity_id, + 1, + KeyIndexBackfillFence { + key_set_signature: repair_signature, + contract_revision: repair_revision, + expected_journal_sequence: 0, + expected_entity_live: true, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: 1, + state: b"snapshot-only", + }), + }, + std::slice::from_ref(&snapshot_key), + ) + .await; + assert!(matches!( + stale, + Err(PersistenceError::JournalBoundaryChanged { + expected: 0, + actual: 1, + }) + )); + assert_eq!( + store + .lookup_by_key(&tenant, entity_type, "path", &journal_key.key_hash) + .await + .expect("lookup current journal ownership"), + Some(entity_id.to_string()) + ); + assert_eq!( + store + .lookup_by_key(&tenant, entity_type, "path", &snapshot_key.key_hash) + .await + .expect("lookup rejected snapshot ownership"), + None + ); + }); +} + +#[test] +fn same_sequence_snapshot_content_rewrite_rejects_repair() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => return, + }; + sqlx::test_block_on(async { + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect to Postgres"); + temper_store_postgres::migration::run_migrations(&pool) + .await + .expect("run Postgres migrations"); + let store = PostgresEventStore::new(pool.clone()); + let tenant = format!("arn238-snapshot-content-fence-{}", sim_uuid()); + let entity_type = "Doc"; + let entity_id = "same-sequence-rewrite"; + let persistence_id = format!("{tenant}:{entity_type}:{entity_id}"); + let repair_signature = "v4:path"; + let captured_snapshot = br#"{"fields":{"WorkspaceId":"before"}}"#; + let replacement_snapshot = br#"{"fields":{"WorkspaceId":"after"}}"#; + let timestamp = sim_now(); + + store + .append( + &persistence_id, + 0, + &[PersistenceEnvelope { + sequence_nr: 1, + event_type: "Create".to_string(), + payload: serde_json::json!({}), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: persistence_id.clone(), + }, + }], + ) + .await + .expect("seed journal generation"); + store + .save_snapshot(&persistence_id, 1, captured_snapshot) + .await + .expect("seed captured snapshot bytes"); + let repair_revision = store + .begin_key_index_backfill(&tenant, entity_type, repair_signature) + .await + .expect("begin snapshot-derived repair"); + + let updated = sqlx::query( + "UPDATE snapshots SET state = $4, created_at = now() \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(&tenant) + .bind(entity_type) + .bind(entity_id) + .bind(replacement_snapshot.as_slice()) + .execute(&pool) + .await + .expect("replace persisted snapshot bytes outside the store writer"); + assert_eq!(updated.rows_affected(), 1); + + let result = store + .backfill_entity_keys( + &tenant, + entity_type, + entity_id, + 1, + KeyIndexBackfillFence { + key_set_signature: repair_signature, + contract_revision: repair_revision, + expected_journal_sequence: 1, + expected_entity_live: true, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: 1, + state: captured_snapshot, + }), + }, + &[], + ) + .await; + assert!(matches!( + result, + Err(PersistenceError::SnapshotGenerationChanged) + )); + }); +} diff --git a/crates/temper-store-postgres/tests/lifecycle_payload_parity.rs b/crates/temper-store-postgres/tests/lifecycle_payload_parity.rs new file mode 100644 index 000000000..892b19271 --- /dev/null +++ b/crates/temper-store-postgres/tests/lifecycle_payload_parity.rs @@ -0,0 +1,54 @@ +//! PostgreSQL lifecycle classification must match the Rust event predicate. + +use temper_runtime::persistence::{EventMetadata, EventStore, PersistenceEnvelope}; +use temper_runtime::scheduler::{sim_now, sim_uuid}; +use temper_store_postgres::PostgresEventStore; + +#[test] +fn legacy_deleted_name_in_array_payload_remains_terminal() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => return, + }; + sqlx::test_block_on(async { + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect to Postgres"); + temper_store_postgres::migration::run_migrations(&pool) + .await + .expect("run Postgres migrations"); + let store = PostgresEventStore::new(pool); + let tenant = format!("arn238-lifecycle-array-{}", sim_uuid()); + let persistence_id = format!("{tenant}:Doc:legacy-array-delete"); + let timestamp = sim_now(); + + store + .append( + &persistence_id, + 0, + &[PersistenceEnvelope { + sequence_nr: 0, + event_type: "Deleted".to_string(), + payload: serde_json::json!(["to_status"]), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: persistence_id.clone(), + }, + }], + ) + .await + .expect("append legacy tombstone with array payload"); + + let entity_ids = store + .list_entity_ids_by_type(&tenant, "Doc") + .await + .expect("list live entities"); + assert!( + entity_ids.is_empty(), + "top-level array membership is not structured to_status metadata" + ); + }); +} diff --git a/crates/temper-store-postgres/tests/snapshot_coverage_invalidation.rs b/crates/temper-store-postgres/tests/snapshot_coverage_invalidation.rs new file mode 100644 index 000000000..c67102b5b --- /dev/null +++ b/crates/temper-store-postgres/tests/snapshot_coverage_invalidation.rs @@ -0,0 +1,1146 @@ +//! PostgreSQL regression for snapshot baseline changes after coverage publication. + +use temper_runtime::persistence::{ + EntityKeyRow, EventMetadata, EventStore, IndexReconciliation, KeyIndexBackfillFence, + PersistenceAppend, PersistenceEnvelope, ProjectionSourceFence, SnapshotBackfillFence, + SnapshotSourceFence, +}; +use temper_runtime::scheduler::{sim_now, sim_uuid}; +use temper_store_postgres::PostgresEventStore; + +type SegmentTopologyRow = (i64, i64, Option, Option, i64, bool); + +#[test] +fn database_triggers_fence_source_writes_from_pre_ledger_binaries() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => return, + }; + sqlx::test_block_on(async { + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect to Postgres"); + temper_store_postgres::migration::run_migrations(&pool) + .await + .expect("run Postgres migrations"); + let tenant = format!("arn238-mixed-version-source-{}", sim_uuid()); + let entity_type = "Doc"; + let event_id = "legacy-event-writer"; + let snapshot_id = "legacy-snapshot-writer"; + + sqlx::query( + "INSERT INTO events \ + (tenant, entity_type, entity_id, sequence_nr, event_type, payload, metadata) \ + VALUES ($1, $2, $3, 1, 'Created', '{}'::jsonb, '{}'::jsonb)", + ) + .bind(&tenant) + .bind(entity_type) + .bind(event_id) + .execute(&pool) + .await + .expect("simulate an event append from a pre-ledger binary"); + assert!( + sqlx::query_scalar::<_, bool>( + "SELECT EXISTS(SELECT 1 FROM query_projection_dirty \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3)", + ) + .bind(&tenant) + .bind(entity_type) + .bind(event_id) + .fetch_one(&pool) + .await + .expect("read event-trigger marker"), + "the database trigger must fence event writes from older binaries" + ); + + sqlx::query( + "DELETE FROM query_projection_dirty \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3", + ) + .bind(&tenant) + .bind(entity_type) + .bind(event_id) + .execute(&pool) + .await + .expect("simulate a new reader clearing the source marker"); + sqlx::query( + "INSERT INTO entity_catalog \ + (tenant, entity_type, entity_id, status, fields, sequence_nr) \ + VALUES ($1, $2, $3, 'Live', '{\"Version\":\"stale\"}'::jsonb, 1)", + ) + .bind(&tenant) + .bind(entity_type) + .bind(event_id) + .execute(&pool) + .await + .expect("simulate a delayed projector from a pre-ledger binary"); + assert!( + sqlx::query_scalar::<_, bool>( + "SELECT EXISTS(SELECT 1 FROM query_projection_dirty \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3)", + ) + .bind(&tenant) + .bind(entity_type) + .bind(event_id) + .fetch_one(&pool) + .await + .expect("read delayed-projector marker"), + "the catalog trigger must re-fence a delayed projector from an older binary" + ); + + sqlx::query( + "INSERT INTO snapshots \ + (tenant, entity_type, entity_id, sequence_nr, state) \ + VALUES ($1, $2, $3, 4, $4)", + ) + .bind(&tenant) + .bind(entity_type) + .bind(snapshot_id) + .bind(b"legacy-snapshot".as_slice()) + .execute(&pool) + .await + .expect("simulate a snapshot write from a pre-ledger binary"); + assert!( + sqlx::query_scalar::<_, bool>( + "SELECT EXISTS(SELECT 1 FROM query_projection_dirty \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3)", + ) + .bind(&tenant) + .bind(entity_type) + .bind(snapshot_id) + .fetch_one(&pool) + .await + .expect("read snapshot-trigger marker"), + "the database trigger must fence snapshot writes from older binaries" + ); + }); +} + +#[test] +fn older_snapshot_writer_cannot_regress_current_generation_or_coverage() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => return, + }; + sqlx::test_block_on(async { + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect to Postgres"); + temper_store_postgres::migration::run_migrations(&pool) + .await + .expect("run Postgres migrations"); + let store = PostgresEventStore::new(pool.clone()); + let tenant = format!("arn238-snapshot-writer-race-{}", sim_uuid()); + let entity_type = "Doc"; + let persistence_id = format!("{tenant}:{entity_type}:delayed-older"); + let signature = "v4:path"; + store + .save_snapshot(&persistence_id, 10, b"newer") + .await + .expect("commit newer snapshot writer"); + let revision = store + .begin_key_index_backfill(&tenant, entity_type, signature) + .await + .expect("begin coverage epoch"); + assert!( + store + .mark_key_index_backfilled_if_revision(&tenant, entity_type, signature, revision,) + .await + .expect("publish coverage") + ); + + store + .save_snapshot(&persistence_id, 5, b"delayed-older") + .await + .expect("complete delayed older snapshot writer"); + + assert_eq!( + store + .load_snapshot(&persistence_id) + .await + .expect("load current snapshot"), + Some((10, b"newer".to_vec())), + "a delayed writer must not replace a newer authoritative snapshot" + ); + assert_eq!( + store + .key_index_reconciliation_revision(&tenant, entity_type) + .await + .expect("read unchanged coverage epoch"), + revision, + "an ignored older snapshot must not invalidate coverage" + ); + assert_eq!( + store + .key_index_backfilled_types(&tenant) + .await + .expect("read preserved coverage watermark"), + vec![(entity_type.to_string(), signature.to_string())] + ); + }); +} + +#[test] +fn exact_snapshot_projection_repair_preserves_published_key_coverage() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => return, + }; + sqlx::test_block_on(async { + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect to Postgres"); + temper_store_postgres::migration::run_migrations(&pool) + .await + .expect("run Postgres migrations"); + let store = PostgresEventStore::new(pool); + let tenant = format!("arn238-snapshot-projection-coverage-{}", sim_uuid()); + let entity_type = "Doc"; + let entity_id = "snapshot-projection"; + let persistence_id = format!("{tenant}:{entity_type}:{entity_id}"); + let signature = "v4:path"; + let snapshot = b"snapshot-only-generation"; + let fields = serde_json::json!({"Id": entity_id, "Path": "/snapshot"}); + let state = serde_json::json!({ + "entity_type": entity_type, + "entity_id": entity_id, + "status": "Live", + "fields": fields, + "sequence_nr": 5 + }); + let key = EntityKeyRow { + key_name: "path".to_string(), + key_hash: format!("snapshot-projection-key-{}", sim_uuid()), + }; + + store + .save_snapshot(&persistence_id, 5, snapshot) + .await + .expect("seed snapshot-only source"); + let revision = store + .begin_key_index_backfill(&tenant, entity_type, signature) + .await + .expect("begin snapshot-derived key repair"); + store + .backfill_entity_keys( + &tenant, + entity_type, + entity_id, + 5, + KeyIndexBackfillFence { + key_set_signature: signature, + contract_revision: revision, + expected_journal_sequence: 0, + expected_entity_live: true, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: 5, + state: snapshot, + }), + }, + std::slice::from_ref(&key), + ) + .await + .expect("repair snapshot-derived key ownership"); + assert!( + store + .mark_key_index_backfilled_if_revision(&tenant, entity_type, signature, revision,) + .await + .expect("publish key coverage") + ); + + assert!( + store + .upsert_query_projection_with_state_if_source( + &tenant, + entity_type, + entity_id, + "Live", + &fields, + &state, + 5, + ProjectionSourceFence { + expected_journal_sequence: 0, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: 5, + state: snapshot, + }), + }, + ) + .await + .expect("repair query projection from the exact snapshot source") + ); + + assert_eq!( + store + .key_index_reconciliation_revision(&tenant, entity_type) + .await + .expect("read preserved key revision"), + revision, + "derived projection repair must not invalidate snapshot-derived ownership" + ); + assert_eq!( + store + .key_index_backfilled_types(&tenant) + .await + .expect("read preserved key coverage"), + vec![(entity_type.to_string(), signature.to_string())] + ); + assert_eq!( + store + .lookup_by_key(&tenant, entity_type, "path", &key.key_hash) + .await + .expect("resolve preserved snapshot-derived key"), + Some(entity_id.to_string()) + ); + }); +} + +#[test] +fn same_sequence_snapshot_rewrite_invalidates_published_key_coverage() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => return, + }; + sqlx::test_block_on(async { + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect to Postgres"); + temper_store_postgres::migration::run_migrations(&pool) + .await + .expect("run Postgres migrations"); + let store = PostgresEventStore::new(pool); + let tenant = format!("arn238-snapshot-published-{}", sim_uuid()); + let entity_type = "Doc"; + let persistence_id = format!("{tenant}:{entity_type}:snapshot-rewrite"); + let signature = "v4:path"; + let timestamp = sim_now(); + store + .append( + &persistence_id, + 0, + &[PersistenceEnvelope { + sequence_nr: 0, + event_type: "Create".to_string(), + payload: serde_json::json!({}), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: persistence_id.clone(), + }, + }], + ) + .await + .expect("seed journal high-water"); + store + .save_snapshot(&persistence_id, 1, b"before") + .await + .expect("seed captured snapshot baseline"); + let revision = store + .begin_key_index_backfill(&tenant, entity_type, signature) + .await + .expect("begin coverage epoch"); + assert!( + store + .mark_key_index_backfilled_if_revision(&tenant, entity_type, signature, revision,) + .await + .expect("publish coverage") + ); + + store + .save_snapshot(&persistence_id, 1, b"before") + .await + .expect("repeat identical snapshot write"); + assert_eq!( + store + .key_index_reconciliation_revision(&tenant, entity_type) + .await + .expect("read unchanged snapshot coverage epoch"), + revision, + "identical snapshot bytes and sequence must not churn the coverage epoch" + ); + assert_eq!( + store + .key_index_backfilled_types(&tenant) + .await + .expect("read preserved snapshot coverage watermark"), + vec![(entity_type.to_string(), signature.to_string())], + "identical snapshot writes must preserve published coverage" + ); + + store + .upsert_query_projection( + &tenant, + entity_type, + "snapshot-rewrite", + "Ready", + &serde_json::json!({"Path": "/journal-dominated"}), + 1, + ) + .await + .expect("write catalog projection represented by the journal"); + assert_eq!( + store + .key_index_reconciliation_revision(&tenant, entity_type) + .await + .expect("read unchanged catalog coverage epoch"), + revision, + "a catalog projection at the journal high-water must reuse the append fence" + ); + assert_eq!( + store + .key_index_backfilled_types(&tenant) + .await + .expect("read preserved catalog coverage watermark"), + vec![(entity_type.to_string(), signature.to_string())], + "journal-dominated catalog writes must preserve published coverage" + ); + + store + .save_snapshot(&persistence_id, 1, b"after") + .await + .expect("rewrite snapshot bytes at the journal high-water"); + + assert!( + store + .key_index_reconciliation_revision(&tenant, entity_type) + .await + .expect("read current coverage epoch") + > revision, + "changed snapshot baseline bytes must invalidate the published coverage epoch" + ); + assert!( + store + .key_index_backfilled_types(&tenant) + .await + .expect("read coverage watermarks") + .is_empty(), + "stale ownership rows must not remain authoritative after a snapshot rewrite" + ); + }); +} + +#[test] +fn projection_repair_fences_snapshot_bytes_and_survives_delayed_plain_delivery() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => return, + }; + sqlx::test_block_on(async { + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect to Postgres"); + temper_store_postgres::migration::run_migrations(&pool) + .await + .expect("run Postgres migrations"); + let store = PostgresEventStore::new(pool.clone()); + let tenant = format!("arn238-projection-source-{}", sim_uuid()); + let entity_type = "Doc"; + let entity_id = "projection-race"; + let persistence_id = format!("{tenant}:{entity_type}:{entity_id}"); + let snapshot_a = b"snapshot-a".to_vec(); + let snapshot_b = b"snapshot-b".to_vec(); + let fields_a = serde_json::json!({"Id": entity_id, "Status": "Live", "Version": "A"}); + let state_a = fields_a.clone(); + let fields_b = serde_json::json!({"Id": entity_id, "Status": "Live", "Version": "B"}); + let state_b = fields_b.clone(); + let envelope = |event_type: &str, to_status: &str| PersistenceEnvelope { + sequence_nr: 0, + event_type: event_type.to_string(), + payload: serde_json::json!({"to_status": to_status}), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: sim_now(), + actor_id: persistence_id.clone(), + }, + }; + + store + .append(&persistence_id, 0, &[envelope("Created", "Live")]) + .await + .expect("seed journal generation"); + store + .save_snapshot(&persistence_id, 1, &snapshot_a) + .await + .expect("seed snapshot A"); + assert!( + store + .upsert_query_projection_with_state_if_source( + &tenant, + entity_type, + entity_id, + "Live", + &fields_a, + &state_a, + 1, + ProjectionSourceFence { + expected_journal_sequence: 1, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: 1, + state: &snapshot_a, + }), + }, + ) + .await + .expect("project snapshot A") + ); + store + .save_snapshot(&persistence_id, 1, &snapshot_b) + .await + .expect("replace snapshot A with B at the same HWM"); + assert!( + !store + .upsert_query_projection_with_state_if_source( + &tenant, + entity_type, + entity_id, + "Live", + &fields_a, + &state_a, + 1, + ProjectionSourceFence { + expected_journal_sequence: 1, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: 1, + state: &snapshot_a, + }), + }, + ) + .await + .expect("reject stale snapshot A fence") + ); + assert_eq!( + store + .dirty_query_projection_entity_ids(&tenant, entity_type, 10) + .await + .expect("snapshot B remains dirty"), + vec![entity_id.to_string()] + ); + assert!( + store + .upsert_query_projection_with_state_if_source( + &tenant, + entity_type, + entity_id, + "Live", + &fields_b, + &state_b, + 1, + ProjectionSourceFence { + expected_journal_sequence: 1, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: 1, + state: &snapshot_b, + }), + }, + ) + .await + .expect("repair snapshot B") + ); + + store + .upsert_query_projection_with_state( + &tenant, + entity_type, + entity_id, + "Live", + &fields_a, + &state_a, + 1, + ) + .await + .expect("deliver delayed unfenced snapshot A projection"); + assert_eq!( + store + .dirty_query_projection_entity_ids(&tenant, entity_type, 10) + .await + .expect("delayed projection re-marks dirty"), + vec![entity_id.to_string()] + ); + assert!( + store + .upsert_query_projection_with_state_if_source( + &tenant, + entity_type, + entity_id, + "Live", + &fields_b, + &state_b, + 1, + ProjectionSourceFence { + expected_journal_sequence: 1, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: 1, + state: &snapshot_b, + }), + }, + ) + .await + .expect("repair delayed A back to B") + ); + assert!( + !store + .remove_query_projection_if_exact( + &tenant, + entity_type, + entity_id, + "Live", + &fields_a, + &state_a, + 1, + ) + .await + .expect("full-row cleanup CAS preserves B") + ); + let row = store + .load_entity_catalog_rows_pg(&tenant, entity_type, &[entity_id.to_string()]) + .await + .expect("load repaired catalog") + .pop() + .expect("B row remains"); + assert_eq!(row.fields, fields_b); + let indexed_ids = sqlx::query_scalar::<_, String>( + "SELECT entity_id FROM entity_field_index \ + WHERE tenant = $1 AND entity_type = $2 \ + AND field_name = $3 AND field_value = $4 \ + ORDER BY entity_id", + ) + .bind(&tenant) + .bind(entity_type) + .bind("Version") + .bind("B") + .fetch_all(&pool) + .await + .expect("query B field index"); + assert!( + indexed_ids.contains(&entity_id.to_string()), + "failed exact cleanup must preserve B's EAV rows" + ); + + store + .append(&persistence_id, 1, &[envelope("Delete", "Deleted")]) + .await + .expect("advance source to Deleted"); + assert!( + store + .remove_query_projection_if_source( + &tenant, + entity_type, + entity_id, + ProjectionSourceFence { + expected_journal_sequence: 2, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: 1, + state: &snapshot_b, + }), + }, + ) + .await + .expect("remove Deleted projection") + ); + store + .append(&persistence_id, 2, &[envelope("Restore", "Live")]) + .await + .expect("advance source back to Live"); + assert!( + store + .upsert_query_projection_with_state_if_source( + &tenant, + entity_type, + entity_id, + "Live", + &fields_b, + &state_b, + 3, + ProjectionSourceFence { + expected_journal_sequence: 3, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: 1, + state: &snapshot_b, + }), + }, + ) + .await + .expect("restore Live projection") + ); + assert!( + store + .dirty_query_projection_entity_ids(&tenant, entity_type, 1) + .await + .expect("restored source is clean") + .is_empty() + ); + + assert!( + store + .remove_query_projection_if_exact( + &tenant, + entity_type, + entity_id, + "Live", + &fields_b, + &state_b, + 3, + ) + .await + .expect("remove exact attempted projection after closing fault") + ); + assert_eq!( + store + .dirty_query_projection_entity_ids(&tenant, entity_type, 1) + .await + .expect("exact cleanup marks projection dirty"), + vec![entity_id.to_string()] + ); + assert!( + store + .upsert_query_projection_with_state_if_source( + &tenant, + entity_type, + entity_id, + "Live", + &fields_b, + &state_b, + 3, + ProjectionSourceFence { + expected_journal_sequence: 3, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: 1, + state: &snapshot_b, + }), + }, + ) + .await + .expect("repair exact-cleanup absence") + ); + assert!( + store + .dirty_query_projection_entity_ids(&tenant, entity_type, 1) + .await + .expect("exact-cleanup repair clears marker") + .is_empty() + ); + }); +} + +#[test] +fn exact_snapshot_source_replaces_a_higher_stale_catalog_sequence() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => return, + }; + sqlx::test_block_on(async { + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect to Postgres"); + temper_store_postgres::migration::run_migrations(&pool) + .await + .expect("run Postgres migrations"); + let store = PostgresEventStore::new(pool); + let tenant = format!("arn238-snapshot-catalog-fence-{}", sim_uuid()); + let entity_type = "Doc"; + let entity_id = "snapshot-owner"; + let persistence_id = format!("{tenant}:{entity_type}:{entity_id}"); + let stale_fields = + serde_json::json!({"Id": entity_id, "Status": "Live", "Version": "stale"}); + let current_fields = + serde_json::json!({"Id": entity_id, "Status": "Live", "Version": "snapshot"}); + let snapshot = b"authoritative-snapshot".to_vec(); + + store + .upsert_query_projection_with_state( + &tenant, + entity_type, + entity_id, + "Live", + &stale_fields, + &stale_fields, + 10, + ) + .await + .expect("seed higher catalog-only compatibility sequence"); + store + .save_snapshot(&persistence_id, 5, &snapshot) + .await + .expect("publish lower authoritative snapshot generation"); + assert!( + store + .upsert_query_projection_with_state_if_source( + &tenant, + entity_type, + entity_id, + "Live", + ¤t_fields, + ¤t_fields, + 5, + ProjectionSourceFence { + expected_journal_sequence: 0, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: 5, + state: &snapshot, + }), + }, + ) + .await + .expect("repair from exact snapshot source"), + "the exact current snapshot must outrank a higher stale catalog sequence" + ); + let row = store + .load_entity_catalog_rows_pg(&tenant, entity_type, &[entity_id.to_string()]) + .await + .expect("load repaired catalog") + .pop() + .expect("repaired catalog row"); + assert_eq!(row.sequence_nr, 5); + assert_eq!(row.fields, current_fields); + assert!( + store + .dirty_query_projection_entity_ids(&tenant, entity_type, 1) + .await + .expect("read closed dirty ledger") + .is_empty() + ); + }); +} + +#[test] +fn delayed_snapshot_splits_durable_tail_and_equal_rewrite_preserves_topology() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => return, + }; + sqlx::test_block_on(async { + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect to Postgres"); + temper_store_postgres::migration::run_migrations(&pool) + .await + .expect("run Postgres migrations"); + let store = PostgresEventStore::new(pool.clone()); + let tenant = format!("arn238-delayed-segment-snapshot-{}", sim_uuid()); + let entity_type = "Doc"; + let entity_id = "delayed-segment-snapshot"; + let persistence_id = format!("{tenant}:{entity_type}:{entity_id}"); + let timestamp = sim_now(); + let events = (1..=10) + .map(|sequence| PersistenceEnvelope { + sequence_nr: sequence, + event_type: "Updated".to_string(), + payload: serde_json::json!({"sequence": sequence}), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp, + actor_id: persistence_id.clone(), + }, + }) + .collect::>(); + store + .append(&persistence_id, 0, &events) + .await + .expect("append durable tail before delayed snapshot"); + + store + .save_snapshot(&persistence_id, 5, b"snapshot-a") + .await + .expect("save delayed snapshot boundary"); + + let segment_rows = || async { + sqlx::query_as::<_, (i64, i64, Option, Option, i64, bool)>( + "SELECT segment_index, start_sequence_nr, end_sequence_nr, \ + snapshot_sequence, event_count, sealed_at IS NOT NULL \ + FROM event_segments \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ + ORDER BY segment_index", + ) + .bind(&tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_all(&pool) + .await + .expect("read event segment topology") + }; + let event_rows = || async { + sqlx::query_as::<_, (i64, i64)>( + "SELECT sequence_nr, segment_index FROM events \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ + ORDER BY sequence_nr", + ) + .bind(&tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_all(&pool) + .await + .expect("read event segment assignments") + }; + let segment_versions = || async { + sqlx::query_as::<_, (i64, String)>( + "SELECT segment_index, xmin::text FROM event_segments \ + WHERE tenant = $1 AND entity_type = $2 AND entity_id = $3 \ + ORDER BY segment_index", + ) + .bind(&tenant) + .bind(entity_type) + .bind(entity_id) + .fetch_all(&pool) + .await + .expect("read event segment row versions") + }; + + let expected_segments = vec![ + (0, 1, Some(5), Some(5), 5, true), + (1, 6, Some(10), None, 5, false), + ]; + assert_eq!( + segment_rows().await, + expected_segments, + "a delayed snapshot must retain the already-durable tail in the open successor" + ); + let expected_events = (1..=10) + .map(|sequence| (sequence, if sequence <= 5 { 0 } else { 1 })) + .collect::>(); + assert_eq!( + event_rows().await, + expected_events, + "events after the delayed boundary must move to the successor segment" + ); + + let segments_before_rewrite = segment_rows().await; + let events_before_rewrite = event_rows().await; + let segment_versions_before_rewrite = segment_versions().await; + store + .save_snapshot(&persistence_id, 5, b"snapshot-b") + .await + .expect("replace same-sequence snapshot bytes"); + assert_eq!(segment_rows().await, segments_before_rewrite); + assert_eq!(event_rows().await, events_before_rewrite); + assert_eq!( + segment_versions().await, + segment_versions_before_rewrite, + "same-sequence source replacement must not rewrite segment rows" + ); + assert_eq!( + store + .load_snapshot(&persistence_id) + .await + .expect("load replaced snapshot"), + Some((5, b"snapshot-b".to_vec())) + ); + }); +} + +#[test] +fn snapshot_only_and_batch_appends_keep_real_segments_contiguous() { + let database_url = match std::env::var("DATABASE_URL") { + Ok(url) => url, + Err(_) => return, + }; + sqlx::test_block_on(async { + let pool = sqlx::PgPool::connect(&database_url) + .await + .expect("connect to Postgres"); + temper_store_postgres::migration::run_migrations(&pool) + .await + .expect("run Postgres migrations"); + let store = PostgresEventStore::new(pool.clone()); + let tenant = format!("arn238-snapshot-only-segments-{}", sim_uuid()); + let envelope = |persistence_id: &str, event_type: &str| PersistenceEnvelope { + sequence_nr: 0, + event_type: event_type.to_string(), + payload: serde_json::json!({}), + metadata: EventMetadata { + event_id: sim_uuid(), + causation_id: sim_uuid(), + correlation_id: sim_uuid(), + timestamp: sim_now(), + actor_id: persistence_id.to_string(), + }, + }; + + let snapshot_only_id = "snapshot-only-first-journal"; + let snapshot_only_pid = format!("{tenant}:Doc:{snapshot_only_id}"); + let snapshot = b"snapshot-only".to_vec(); + store + .save_snapshot(&snapshot_only_pid, 5, &snapshot) + .await + .expect("save snapshot-only generation"); + let segment_count: (i64,) = sqlx::query_as( + "SELECT COUNT(*) FROM event_segments \ + WHERE tenant = $1 AND entity_type = 'Doc' AND entity_id = $2", + ) + .bind(&tenant) + .bind(snapshot_only_id) + .fetch_one(&pool) + .await + .expect("count snapshot-only segments"); + assert_eq!(segment_count.0, 0); + store + .append_with_index_rows( + &snapshot_only_pid, + 0, + &[envelope(&snapshot_only_pid, "FirstJournalEvent")], + &[], + &[], + IndexReconciliation { + snapshot_source: SnapshotSourceFence::Exact { + sequence_nr: 5, + state: snapshot, + }, + ..IndexReconciliation::default() + }, + ) + .await + .expect("append first journal generation over snapshot baseline"); + let first_topology: Vec<(i64, i64, Option, i64, bool)> = sqlx::query_as( + "SELECT segment_index, start_sequence_nr, end_sequence_nr, event_count, sealed_at IS NOT NULL \ + FROM event_segments WHERE tenant = $1 AND entity_type = 'Doc' AND entity_id = $2 \ + ORDER BY segment_index", + ) + .bind(&tenant) + .bind(snapshot_only_id) + .fetch_all(&pool) + .await + .expect("load first journal topology"); + assert_eq!(first_topology, vec![(0, 1, Some(1), 1, false)]); + + let batch_id = "batch-after-snapshot"; + let batch_pid = format!("{tenant}:Doc:{batch_id}"); + store + .append(&batch_pid, 0, &[envelope(&batch_pid, "Created")]) + .await + .expect("seed batch journal"); + store + .save_snapshot(&batch_pid, 1, b"snapshot-1") + .await + .expect("seal first batch segment"); + store + .append_batch(&[PersistenceAppend { + persistence_id: batch_pid.clone(), + expected_sequence: 1, + events: vec![envelope(&batch_pid, "Batched")], + key_rows: Vec::new(), + reconcile_keys: false, + key_set_signature: None, + snapshot_source: SnapshotSourceFence::Exact { + sequence_nr: 1, + state: b"snapshot-1".to_vec(), + }, + batch_idempotency: None, + }]) + .await + .expect("append batch into open successor"); + let batch_topology: Vec<(i64, i64, Option, i64, bool)> = sqlx::query_as( + "SELECT segment_index, start_sequence_nr, end_sequence_nr, event_count, sealed_at IS NOT NULL \ + FROM event_segments WHERE tenant = $1 AND entity_type = 'Doc' AND entity_id = $2 \ + ORDER BY segment_index", + ) + .bind(&tenant) + .bind(batch_id) + .fetch_all(&pool) + .await + .expect("load batch topology"); + assert_eq!( + batch_topology, + vec![(0, 1, Some(1), 1, true), (1, 2, Some(2), 1, false)] + ); + let assignments: Vec<(i64, i64)> = sqlx::query_as( + "SELECT sequence_nr, segment_index FROM events \ + WHERE tenant = $1 AND entity_type = 'Doc' AND entity_id = $2 ORDER BY sequence_nr", + ) + .bind(&tenant) + .bind(batch_id) + .fetch_all(&pool) + .await + .expect("load batch assignments"); + assert_eq!(assignments, vec![(1, 0), (2, 1)]); + + let ahead_id = "snapshot-ahead-of-journal"; + let ahead_pid = format!("{tenant}:Doc:{ahead_id}"); + store + .append( + &ahead_pid, + 0, + &[ + envelope(&ahead_pid, "Created"), + envelope(&ahead_pid, "Updated"), + ], + ) + .await + .expect("seed journal below migration snapshot"); + let ahead_snapshot = b"snapshot-5".to_vec(); + store + .save_snapshot(&ahead_pid, 5, &ahead_snapshot) + .await + .expect("save migration snapshot ahead of journal HWM 2"); + let topology_before_legacy_ghost: Vec = + sqlx::query_as( + "SELECT segment_index, start_sequence_nr, end_sequence_nr, snapshot_sequence, \ + event_count, sealed_at IS NOT NULL FROM event_segments \ + WHERE tenant = $1 AND entity_type = 'Doc' AND entity_id = $2 ORDER BY segment_index", + ) + .bind(&tenant) + .bind(ahead_id) + .fetch_all(&pool) + .await + .expect("load topology after snapshot ahead of journal"); + assert_eq!( + topology_before_legacy_ghost, + vec![(0, 1, Some(2), None, 2, false)], + "snapshot sequence 5 must not manufacture a boundary beyond journal HWM 2" + ); + + sqlx::query( + "INSERT INTO event_segments \ + (tenant, entity_type, entity_id, segment_index, start_sequence_nr, event_count) \ + VALUES ($1, 'Doc', $2, 1, 6, 0)", + ) + .bind(&tenant) + .bind(ahead_id) + .execute(&pool) + .await + .expect("seed legacy future-start segment"); + store + .append_with_index_rows( + &ahead_pid, + 2, + &[envelope(&ahead_pid, "AfterMigrationSnapshot")], + &[], + &[], + IndexReconciliation { + snapshot_source: SnapshotSourceFence::Exact { + sequence_nr: 5, + state: ahead_snapshot, + }, + ..IndexReconciliation::default() + }, + ) + .await + .expect("append repairs legacy future-start topology"); + let repaired_topology: Vec = + sqlx::query_as( + "SELECT segment_index, start_sequence_nr, end_sequence_nr, snapshot_sequence, \ + event_count, sealed_at IS NOT NULL FROM event_segments \ + WHERE tenant = $1 AND entity_type = 'Doc' AND entity_id = $2 ORDER BY segment_index", + ) + .bind(&tenant) + .bind(ahead_id) + .fetch_all(&pool) + .await + .expect("load repaired topology"); + assert_eq!(repaired_topology, vec![(0, 1, Some(3), None, 3, false)]); + let repaired_assignments: Vec<(i64, i64)> = sqlx::query_as( + "SELECT sequence_nr, segment_index FROM events \ + WHERE tenant = $1 AND entity_type = 'Doc' AND entity_id = $2 ORDER BY sequence_nr", + ) + .bind(&tenant) + .bind(ahead_id) + .fetch_all(&pool) + .await + .expect("load repaired event assignments"); + assert_eq!(repaired_assignments, vec![(1, 0), (2, 0), (3, 0)]); + }); +} diff --git a/crates/temper-store-redis/src/event_store.rs b/crates/temper-store-redis/src/event_store.rs index 2df3f9363..bea02034f 100644 --- a/crates/temper-store-redis/src/event_store.rs +++ b/crates/temper-store-redis/src/event_store.rs @@ -1,4 +1,5 @@ //! Redis-backed implementation of the [`EventStore`] trait. + //! //! Uses Redis primitives: //! - `LIST` per entity for ordered event journal entries @@ -10,53 +11,341 @@ //! check-and-set the sequence number, preventing lost-update races between //! concurrent writers on the same entity. +mod atomic; + +#[cfg(test)] +mod tests; + use std::sync::Arc; use fred::prelude::*; use fred::types::scripts::Script; use serde::{Deserialize, Serialize}; +#[cfg(test)] +use temper_runtime::persistence::STATE_MATERIALIZATION_EVENT_TYPE; use temper_runtime::persistence::{ - EventStore, PersistenceAppend, PersistenceAppendResult, PersistenceEnvelope, PersistenceError, - storage_error, + EventStore, IndexReconciliation, JournalBoundary, PersistenceAppend, PersistenceAppendResult, + PersistenceBatchIdempotency, PersistenceEnvelope, PersistenceError, SnapshotSourceFence, + is_state_materialization_event_for, storage_error, }; +use temper_runtime::scheduler::sim_now; use temper_runtime::tenant::parse_persistence_id_parts; -/// Lua script for atomic append: check expected sequence, RPUSH events, SET new seq, SADD entity ref. +/// Lua script for one atomic journal, entity-catalog, and segment append. /// -/// KEYS[1] = seq_key, KEYS[2] = events_key, KEYS[3] = entities_key -/// ARGV[1] = expected_seq (string-encoded integer) -/// ARGV[2] = entity_ref_json (for SADD into entities set) -/// ARGV[3..N] = serialized event JSONs +/// KEYS: journal sequence, journal events, tenant entities, current-segment +/// pointer, expected active segment, current snapshot, canonical segment zero, +/// the validated materialization-generation marker, first-terminal sequence, +/// and optional batch-idempotency claim. +/// ARGV: expected journal sequence, entity ref, expected segment, timestamp, +/// snapshot-fence mode, exact expected snapshot record, whether to retire the +/// migration snapshot, first terminal sequence in this append (zero if absent), +/// optional batch intent hash, then event records. /// -/// Returns: `{1, new_seq}` on success, `{0, current_seq}` on conflict. +/// Returns `{1, new_seq}` on success, `{0, current_seq}` on journal conflict, +/// `{2, current_segment}` when the caller must retry its pointer capture, or +/// `{3, current_seq}` when the exact snapshot-generation fence changed. const APPEND_LUA: &str = r#" local seq_key = KEYS[1] local events_key = KEYS[2] local entities_key = KEYS[3] +local segment_pointer_key = KEYS[4] +local active_segment_key = KEYS[5] +local snapshot_key = KEYS[6] +local canonical_segment_key = KEYS[7] +local materialization_marker_key = KEYS[8] +local terminal_sequence_key = KEYS[9] +local batch_idempotency_key = KEYS[10] local expected = tonumber(ARGV[1]) local entity_ref = ARGV[2] +local expected_segment = tonumber(ARGV[3]) +local timestamp = ARGV[4] +local snapshot_fence_mode = tonumber(ARGV[5]) +local expected_snapshot = ARGV[6] +local retire_snapshot = tonumber(ARGV[7]) +local batch_intent_hash = ARGV[9] + +local function snapshots_equal(left_json, right_json) + if not left_json or not right_json or right_json == '' then + return false + end + local left = cjson.decode(left_json) + local right = cjson.decode(right_json) + return tonumber(left.sequence_nr) == tonumber(right.sequence_nr) + and cjson.encode(left.snapshot) == cjson.encode(right.snapshot) +end local current = tonumber(redis.call('GET', seq_key) or '0') +if batch_intent_hash ~= '' then + local committed_hash = redis.call('GET', batch_idempotency_key) + if committed_hash then + if committed_hash == batch_intent_hash then + return {4, current} + end + return {5, current} + end +end if current ~= expected then return {0, current} end -for i = 3, #ARGV do +local current_snapshot = redis.call('GET', snapshot_key) +if snapshot_fence_mode == 1 and current_snapshot then + return {3, current} +end +if snapshot_fence_mode == 2 + and not snapshots_equal(current_snapshot, expected_snapshot) then + return {3, current} +end + +local current_segment = tonumber(redis.call('GET', segment_pointer_key) or '0') +if current_segment ~= expected_segment then + return {2, current_segment} +end + +local segment +local target_segment_key +if current == 0 then + current_segment = 0 + target_segment_key = canonical_segment_key + segment = { + segment_index = 0, + start_sequence_nr = 1, + end_sequence_nr = cjson.null, + snapshot_sequence = cjson.null, + event_count = 0, + sealed_at = cjson.null, + created_at = timestamp + } +else + target_segment_key = active_segment_key + local encoded_segment = redis.call('GET', active_segment_key) + if encoded_segment then + segment = cjson.decode(encoded_segment) + local start_sequence = tonumber(segment.start_sequence_nr) + local invalid_end = segment.end_sequence_nr + and segment.end_sequence_nr ~= cjson.null + and tonumber(segment.end_sequence_nr) < start_sequence + if start_sequence > expected + 1 or invalid_end then + -- Repair legacy snapshot-ahead topology from the authoritative + -- journal high-water before assigning the new event. + current_segment = 0 + target_segment_key = canonical_segment_key + segment = { + segment_index = 0, + start_sequence_nr = 1, + end_sequence_nr = expected, + snapshot_sequence = cjson.null, + event_count = expected, + sealed_at = cjson.null, + created_at = timestamp + } + end + else + current_segment = 0 + target_segment_key = canonical_segment_key + segment = { + segment_index = 0, + start_sequence_nr = 1, + end_sequence_nr = expected, + snapshot_sequence = cjson.null, + event_count = expected, + sealed_at = cjson.null, + created_at = timestamp + } + end +end + +for i = 10, #ARGV do redis.call('RPUSH', events_key, ARGV[i]) end -local new_seq = expected + (#ARGV - 2) +local new_seq = expected + (#ARGV - 9) +segment.end_sequence_nr = new_seq +segment.event_count = math.max(new_seq - tonumber(segment.start_sequence_nr) + 1, 0) + redis.call('SET', seq_key, tostring(new_seq)) +local appended_terminal_sequence = tonumber(ARGV[8]) +local stored_terminal_sequence = redis.call('GET', terminal_sequence_key) +if not stored_terminal_sequence and current == 0 then + redis.call('SET', terminal_sequence_key, tostring(appended_terminal_sequence)) +elseif stored_terminal_sequence + and appended_terminal_sequence > 0 + and tonumber(stored_terminal_sequence) == 0 then + redis.call('SET', terminal_sequence_key, tostring(appended_terminal_sequence)) +end redis.call('SADD', entities_key, entity_ref) +if expected_segment ~= current_segment then + redis.call('DEL', active_segment_key) +end +redis.call('SET', target_segment_key, cjson.encode(segment)) +redis.call('SET', segment_pointer_key, tostring(current_segment)) +if retire_snapshot == 1 then + redis.call('DEL', snapshot_key) + redis.call('SET', materialization_marker_key, '1') +end +if batch_intent_hash ~= '' then + redis.call('SET', batch_idempotency_key, batch_intent_hash) +end return {1, new_seq} "#; +/// Atomically read a stream's journal and terminal metadata pair. +const READ_BOUNDARY_LUA: &str = r#" +local latest = redis.call('GET', KEYS[1]) or '0' +local terminal = redis.call('GET', KEYS[2]) or '' +return {latest, terminal} +"#; + +/// Install lazily reconstructed terminal metadata only if the captured journal +/// high-water is still current and no other migrator already installed it. +const BACKFILL_BOUNDARY_LUA: &str = r#" +local current = redis.call('GET', KEYS[1]) or '0' +if current ~= ARGV[1] then + return 0 +end +if redis.call('EXISTS', KEYS[2]) == 0 then + redis.call('SET', KEYS[2], ARGV[2]) +end +return 1 +"#; + +/// Atomically retain only a monotonic current snapshot and rotate its segment. +/// +/// KEYS: current snapshot, history row, segment pointer, active segment, next +/// segment, journal high-water, the validated materialization marker, and the +/// tenant entity set. +/// ARGV: expected segment index, sequence, snapshot JSON, history JSON, timestamp, +/// snapshot-source fence mode, exact expected source record, and entity ref. +/// Returns `0` on older/identical no-op, `1` on an initial/newer write, `2` on +/// equal-sequence byte replacement, or `-1` when the caller must retry with the +/// new segment pointer. A write rotates only when a journal exists; `-2` rejects +/// a changed exact snapshot source. +const SAVE_SNAPSHOT_LUA: &str = r#" +local expected_segment = tonumber(ARGV[1]) +local incoming_sequence = tonumber(ARGV[2]) +local incoming_snapshot = ARGV[3] +local current_snapshot = redis.call('GET', KEYS[1]) +local same_sequence_rewrite = false +local snapshot_fence_mode = tonumber(ARGV[6]) +local expected_snapshot = ARGV[7] + +local function snapshot_payloads_equal(left, right) + return cjson.encode(left.snapshot) == cjson.encode(right.snapshot) +end + +local function snapshot_records_equal(left_json, right_json) + if not left_json or not right_json or right_json == '' then + return false + end + local left = cjson.decode(left_json) + local right = cjson.decode(right_json) + return tonumber(left.sequence_nr) == tonumber(right.sequence_nr) + and snapshot_payloads_equal(left, right) +end + +if snapshot_fence_mode == 1 and current_snapshot then + return -2 +end +if snapshot_fence_mode == 2 + and not snapshot_records_equal(current_snapshot, expected_snapshot) then + return -2 +end + +local journal_sequence = tonumber(redis.call('GET', KEYS[6]) or '0') +if snapshot_fence_mode == 0 + and redis.call('GET', KEYS[7]) == '1' then + return 0 +end + +redis.call('SADD', KEYS[8], ARGV[8]) + +if current_snapshot then + local current = cjson.decode(current_snapshot) + local incoming = cjson.decode(incoming_snapshot) + local current_sequence = tonumber(current.sequence_nr) + if current_sequence > incoming_sequence + or (current_sequence == incoming_sequence and snapshot_payloads_equal(current, incoming)) then + return 0 + end + same_sequence_rewrite = current_sequence == incoming_sequence +end + +if same_sequence_rewrite then + redis.call('SET', KEYS[1], incoming_snapshot) + redis.call('SET', KEYS[2], ARGV[4]) + return 2 +end + +if journal_sequence == 0 then + redis.call('SET', KEYS[1], incoming_snapshot) + redis.call('SET', KEYS[2], ARGV[4]) + return 1 +end +if incoming_sequence > journal_sequence then + redis.call('SET', KEYS[1], incoming_snapshot) + redis.call('SET', KEYS[2], ARGV[4]) + return 1 +end + +local current_segment = tonumber(redis.call('GET', KEYS[3]) or '0') +if current_segment ~= expected_segment then + return -1 +end + +local timestamp = ARGV[5] +local encoded_segment = redis.call('GET', KEYS[4]) +local segment +if encoded_segment then + segment = cjson.decode(encoded_segment) +else + segment = { + segment_index = current_segment, + start_sequence_nr = 1, + end_sequence_nr = cjson.null, + snapshot_sequence = cjson.null, + event_count = 0, + sealed_at = cjson.null, + created_at = timestamp + } +end + +segment.end_sequence_nr = incoming_sequence +segment.snapshot_sequence = incoming_sequence +segment.event_count = math.max(incoming_sequence - tonumber(segment.start_sequence_nr) + 1, 0) +segment.sealed_at = timestamp + +local next_segment = { + segment_index = current_segment + 1, + start_sequence_nr = incoming_sequence + 1, + end_sequence_nr = journal_sequence > incoming_sequence and journal_sequence or cjson.null, + snapshot_sequence = cjson.null, + event_count = math.max(journal_sequence - incoming_sequence, 0), + sealed_at = cjson.null, + created_at = timestamp +} + +redis.call('SET', KEYS[1], incoming_snapshot) +redis.call('SET', KEYS[2], ARGV[4]) +redis.call('SET', KEYS[4], cjson.encode(segment)) +redis.call('SET', KEYS[5], cjson.encode(next_segment)) +redis.call('SET', KEYS[3], tostring(current_segment + 1)) +return 1 +"#; + +const JOURNAL_BOUNDARY_PAGE_SIZE: usize = 1_024; +const APPEND_POINTER_RETRY_BUDGET: usize = 8; +const SNAPSHOT_POINTER_RETRY_BUDGET: usize = 8; + /// Redis-backed event store. #[derive(Clone)] pub struct RedisEventStore { client: Arc, append_script: Script, + snapshot_script: Script, + read_boundary_script: Script, + backfill_boundary_script: Script, } #[derive(Debug, Serialize, Deserialize)] @@ -72,7 +361,8 @@ struct SnapshotHistoryRecord { created_at: chrono::DateTime, } -#[derive(Debug, Serialize, Deserialize)] +#[cfg(test)] +#[derive(Debug, Deserialize)] struct SegmentRecord { segment_index: u64, start_sequence_nr: u64, @@ -100,6 +390,9 @@ impl RedisEventStore { Ok(Self { client: Arc::new(client), append_script: Script::from_lua(APPEND_LUA), + snapshot_script: Script::from_lua(SAVE_SNAPSHOT_LUA), + read_boundary_script: Script::from_lua(READ_BOUNDARY_LUA), + backfill_boundary_script: Script::from_lua(BACKFILL_BOUNDARY_LUA), }) } @@ -122,6 +415,13 @@ impl RedisEventStore { ) } + fn terminal_sequence_key(tenant: &str, entity_type: &str, entity_id: &str) -> String { + format!( + "{}:events_terminal:{tenant}:{entity_type}:{entity_id}", + crate::keys::PREFIX + ) + } + fn snapshot_key(tenant: &str, entity_type: &str, entity_id: &str) -> String { format!( "{}:snapshot:{tenant}:{entity_type}:{entity_id}", @@ -129,6 +429,13 @@ impl RedisEventStore { ) } + fn materialization_marker_key(tenant: &str, entity_type: &str, entity_id: &str) -> String { + format!( + "{}:state_materialized:{tenant}:{entity_type}:{entity_id}", + crate::keys::PREFIX + ) + } + fn snapshot_history_key( tenant: &str, entity_type: &str, @@ -159,6 +466,20 @@ impl RedisEventStore { format!("{}:entities:{tenant}", crate::keys::PREFIX) } + fn batch_idempotency_key(claim: Option<&PersistenceBatchIdempotency>) -> String { + match claim { + Some(claim) => format!( + "{}:batch_idempotency:{}:{}:{}:{}", + crate::keys::PREFIX, + claim.persistence_id.len(), + claim.persistence_id, + claim.idempotency_key.len(), + claim.idempotency_key + ), + None => format!("{}:batch_idempotency:none", crate::keys::PREFIX), + } + } + fn trajectory_key(tenant: &str) -> String { format!("{}:trajectories:{tenant}", crate::keys::PREFIX) } @@ -204,106 +525,59 @@ impl RedisEventStore { } impl EventStore for RedisEventStore { + async fn batch_idempotency_committed( + &self, + claim: &PersistenceBatchIdempotency, + ) -> Result { + let key = Self::batch_idempotency_key(Some(claim)); + let committed_hash: Option = self.client.get(&key).await.map_err(storage_error)?; + let Some(committed_hash) = committed_hash else { + return Ok(false); + }; + if committed_hash != claim.intent_hash { + return Err(PersistenceError::Storage(format!( + "atomic batch idempotency key '{}' was reused with a different intent", + claim.idempotency_key + ))); + } + Ok(true) + } + async fn append( &self, persistence_id: &str, expected_sequence: u64, events: &[PersistenceEnvelope], ) -> Result { - let (tenant, entity_type, entity_id) = - parse_persistence_id_parts(persistence_id).map_err(PersistenceError::Storage)?; - let seq_key = Self::seq_key(tenant, entity_type, entity_id); - let events_key = Self::events_key(tenant, entity_type, entity_id); - let entities_key = Self::tenant_entities_key(tenant); - - // Pre-serialize events with provisional sequence numbers. - let mut args: Vec = Vec::with_capacity(events.len() + 2); - args.push(expected_sequence.to_string()); - - let entity_ref = EntityRef { - entity_type: entity_type.to_string(), - entity_id: entity_id.to_string(), - }; - let entity_ref_json = serde_json::to_string(&entity_ref) - .map_err(|e| PersistenceError::Serialization(e.to_string()))?; - args.push(entity_ref_json); - - let mut seq = expected_sequence; - for event in events { - seq += 1; - let mut env = event.clone(); - env.sequence_nr = seq; - let encoded = serde_json::to_string(&env) - .map_err(|e| PersistenceError::Serialization(e.to_string()))?; - args.push(encoded); - } - - let keys = vec![seq_key, events_key, entities_key]; - let result: Vec = self - .append_script - .evalsha_with_reload(&self.client, keys, args) - .await - .map_err(storage_error)?; + self.append_atomically( + persistence_id, + expected_sequence, + events, + &SnapshotSourceFence::Unchecked, + None, + ) + .await + .map(|(sequence_nr, _)| sequence_nr) + } - match result.as_slice() { - [1, new_seq] => { - let new_seq = *new_seq as u64; - let current_segment_key = Self::current_segment_key(tenant, entity_type, entity_id); - let current_segment_raw: Option = self - .client - .get(¤t_segment_key) - .await - .map_err(storage_error)?; - let segment_index = current_segment_raw - .and_then(|raw| raw.parse::().ok()) - .unwrap_or(0); - let segment_key = Self::segment_key(tenant, entity_type, entity_id, segment_index); - let existing: Option = - self.client.get(&segment_key).await.map_err(storage_error)?; - let mut record = existing - .as_deref() - .map(serde_json::from_str::) - .transpose() - .map_err(|e| PersistenceError::Serialization(e.to_string()))? - .unwrap_or_else(|| SegmentRecord { - segment_index, - start_sequence_nr: (expected_sequence + 1).max(1), - end_sequence_nr: None, - snapshot_sequence: None, - event_count: 0, - sealed_at: None, - created_at: chrono::Utc::now(), - }); - record.end_sequence_nr = Some(new_seq); - record.event_count = new_seq.saturating_sub(record.start_sequence_nr) + 1; - let encoded = serde_json::to_string(&record) - .map_err(|e| PersistenceError::Serialization(e.to_string()))?; - let _: () = self - .client - .set(&segment_key, encoded, None, None, false) - .await - .map_err(storage_error)?; - let _: () = self - .client - .set( - ¤t_segment_key, - segment_index.to_string(), - None, - None, - false, - ) - .await - .map_err(storage_error)?; - Ok(new_seq) - } - [0, actual] => Err(PersistenceError::ConcurrencyViolation { - expected: expected_sequence, - actual: *actual as u64, - }), - other => Err(PersistenceError::Storage(format!( - "unexpected Lua script result: {other:?}" - ))), - } + async fn append_with_index_rows( + &self, + persistence_id: &str, + expected_sequence: u64, + events: &[PersistenceEnvelope], + _key_rows: &[temper_runtime::persistence::EntityKeyRow], + _vector_rows: &[temper_runtime::persistence::EntityVectorRow], + reconciliation: IndexReconciliation, + ) -> Result { + self.append_atomically( + persistence_id, + expected_sequence, + events, + &reconciliation.snapshot_source, + None, + ) + .await + .map(|(sequence_nr, _)| sequence_nr) } async fn append_batch( @@ -313,16 +587,19 @@ impl EventStore for RedisEventStore { match appends { [] => Ok(Vec::new()), [append] => { - let sequence_nr = self - .append( + let (sequence_nr, batch_already_applied) = self + .append_atomically( &append.persistence_id, append.expected_sequence, &append.events, + &append.snapshot_source, + append.batch_idempotency.as_ref(), ) .await?; Ok(vec![PersistenceAppendResult { persistence_id: append.persistence_id.clone(), sequence_nr, + batch_already_applied, }]) } _ => Err(PersistenceError::Storage( @@ -360,109 +637,177 @@ impl EventStore for RedisEventStore { Ok(out) } - async fn save_snapshot( + async fn read_events_page( &self, persistence_id: &str, - sequence_nr: u64, - snapshot: &[u8], - ) -> Result<(), PersistenceError> { + from_sequence: u64, + through_sequence: u64, + limit: usize, + ) -> Result, PersistenceError> { + assert!(limit > 0, "event page limit must be positive"); + assert!( + through_sequence >= from_sequence, + "event page boundary must not precede its cursor" + ); let (tenant, entity_type, entity_id) = parse_persistence_id_parts(persistence_id).map_err(PersistenceError::Storage)?; - let key = Self::snapshot_key(tenant, entity_type, entity_id); - let record = SnapshotRecord { - sequence_nr, - snapshot: snapshot.to_vec(), - }; - let encoded = serde_json::to_string(&record) - .map_err(|e| PersistenceError::Serialization(e.to_string()))?; - let _: () = self + let events_key = Self::events_key(tenant, entity_type, entity_id); + let remaining = through_sequence.saturating_sub(from_sequence); + if remaining == 0 { + return Ok(Vec::new()); + } + let page_len = remaining.min(limit as u64); + let start_index = i64::try_from(from_sequence).map_err(|_| { + PersistenceError::Storage("event page cursor exceeds Redis list index".to_string()) + })?; + let end_sequence = from_sequence.saturating_add(page_len); + let end_index = i64::try_from(end_sequence.saturating_sub(1)).map_err(|_| { + PersistenceError::Storage("event page boundary exceeds Redis list index".to_string()) + })?; + let encoded_events: Vec = self .client - .set(&key, encoded, None, None, false) + .lrange(&events_key, start_index, end_index) .await .map_err(storage_error)?; - let history_key = Self::snapshot_history_key(tenant, entity_type, entity_id, sequence_nr); - let history = SnapshotHistoryRecord { - sequence_nr, - snapshot: snapshot.to_vec(), - created_at: chrono::Utc::now(), - }; - let encoded_history = serde_json::to_string(&history) - .map_err(|e| PersistenceError::Serialization(e.to_string()))?; - let _: () = self - .client - .set(&history_key, encoded_history, None, None, false) - .await - .map_err(storage_error)?; + let mut out = Vec::with_capacity(encoded_events.len()); + for encoded in encoded_events { + let event = serde_json::from_str::(&encoded) + .map_err(|error| PersistenceError::Serialization(error.to_string()))?; + out.push(event); + } + out.sort_by_key(|event| event.sequence_nr); + Ok(out) + } - let current_segment_key = Self::current_segment_key(tenant, entity_type, entity_id); - let current_segment_raw: Option = self - .client - .get(¤t_segment_key) - .await - .map_err(storage_error)?; - let current_segment = current_segment_raw - .and_then(|raw| raw.parse::().ok()) - .unwrap_or(0); - let segment_key = Self::segment_key(tenant, entity_type, entity_id, current_segment); - let existing: Option = - self.client.get(&segment_key).await.map_err(storage_error)?; - let mut segment = existing - .as_deref() - .map(serde_json::from_str::) - .transpose() - .map_err(|e| PersistenceError::Serialization(e.to_string()))? - .unwrap_or_else(|| SegmentRecord { - segment_index: current_segment, - start_sequence_nr: 1, - end_sequence_nr: Some(sequence_nr), - snapshot_sequence: Some(sequence_nr), - event_count: sequence_nr, - sealed_at: None, - created_at: chrono::Utc::now(), - }); - segment.end_sequence_nr = Some(sequence_nr); - segment.snapshot_sequence = Some(sequence_nr); - segment.event_count = sequence_nr.saturating_sub(segment.start_sequence_nr) + 1; - segment.sealed_at = Some(chrono::Utc::now()); - let encoded_segment = serde_json::to_string(&segment) - .map_err(|e| PersistenceError::Serialization(e.to_string()))?; - let _: () = self - .client - .set(&segment_key, encoded_segment, None, None, false) - .await - .map_err(storage_error)?; + async fn journal_boundary( + &self, + persistence_id: &str, + ) -> Result { + let (tenant, entity_type, entity_id) = + parse_persistence_id_parts(persistence_id).map_err(PersistenceError::Storage)?; + let sequence_key = Self::seq_key(tenant, entity_type, entity_id); + let terminal_sequence_key = Self::terminal_sequence_key(tenant, entity_type, entity_id); + + for _attempt in 0..APPEND_POINTER_RETRY_BUDGET { + let encoded: Vec = self + .read_boundary_script + .evalsha_with_reload( + &self.client, + vec![sequence_key.clone(), terminal_sequence_key.clone()], + Vec::::new(), + ) + .await + .map_err(storage_error)?; + let [latest_raw, terminal_raw] = encoded.as_slice() else { + return Err(PersistenceError::Storage(format!( + "unexpected Redis journal boundary result: {encoded:?}" + ))); + }; + let latest_sequence = latest_raw + .parse::() + .map_err(|error| PersistenceError::Serialization(error.to_string()))?; + if !terminal_raw.is_empty() { + let terminal_sequence = terminal_raw + .parse::() + .map_err(|error| PersistenceError::Serialization(error.to_string()))?; + if terminal_sequence > latest_sequence { + return Err(PersistenceError::Storage(format!( + "Redis terminal sequence {terminal_sequence} exceeds journal high-water {latest_sequence} for {persistence_id}" + ))); + } + return Ok(JournalBoundary { + latest_sequence, + first_terminal_sequence: (terminal_sequence > 0).then_some(terminal_sequence), + }); + } - let next_segment = current_segment + 1; - let next_segment_key = Self::segment_key(tenant, entity_type, entity_id, next_segment); - let next = SegmentRecord { - segment_index: next_segment, - start_sequence_nr: sequence_nr + 1, - end_sequence_nr: None, - snapshot_sequence: None, - event_count: 0, - sealed_at: None, - created_at: chrono::Utc::now(), - }; - let encoded_next = serde_json::to_string(&next) - .map_err(|e| PersistenceError::Serialization(e.to_string()))?; - let _: () = self - .client - .set(&next_segment_key, encoded_next, None, None, false) - .await - .map_err(storage_error)?; - let _: () = self - .client - .set( - ¤t_segment_key, - next_segment.to_string(), - None, - None, - false, - ) + // Compatibility migration for streams written before terminal + // metadata existed. Scan only the captured high-water, then install + // the result with a high-water CAS so every later lookup is O(1). + let mut cursor = 0_u64; + let mut first_terminal_sequence = None; + while cursor < latest_sequence && first_terminal_sequence.is_none() { + let remaining = latest_sequence - cursor; + let page_len = usize::try_from(remaining.min(JOURNAL_BOUNDARY_PAGE_SIZE as u64)) + .expect("bounded Redis journal page length fits usize"); + let page = self + .read_events_page(persistence_id, cursor, latest_sequence, page_len) + .await?; + if page.len() != page_len { + return Err(PersistenceError::Storage(format!( + "Redis journal boundary expected {page_len} events after sequence {cursor}, received {}", + page.len() + ))); + } + for (offset, event) in page.iter().enumerate() { + let expected_sequence = cursor + offset as u64 + 1; + if event.sequence_nr != expected_sequence { + return Err(PersistenceError::Storage(format!( + "Redis journal boundary expected sequence {expected_sequence}, received {}", + event.sequence_nr + ))); + } + if event.transitions_to_deleted() { + first_terminal_sequence = Some(event.sequence_nr); + break; + } + } + cursor = page + .last() + .map(|event| event.sequence_nr) + .expect("validated non-empty Redis journal page"); + } + let installed: i64 = self + .backfill_boundary_script + .evalsha_with_reload( + &self.client, + vec![sequence_key.clone(), terminal_sequence_key.clone()], + vec![ + latest_sequence.to_string(), + first_terminal_sequence.unwrap_or(0).to_string(), + ], + ) + .await + .map_err(storage_error)?; + if installed == 1 { + return Ok(JournalBoundary { + latest_sequence, + first_terminal_sequence, + }); + } + } + + Err(PersistenceError::Storage(format!( + "Redis journal boundary for {persistence_id} did not stabilize after {APPEND_POINTER_RETRY_BUDGET} attempts" + ))) + } + + async fn save_snapshot( + &self, + persistence_id: &str, + sequence_nr: u64, + snapshot: &[u8], + ) -> Result<(), PersistenceError> { + self.save_snapshot_atomically( + persistence_id, + sequence_nr, + snapshot, + &SnapshotSourceFence::Unchecked, + ) + .await + } + + async fn save_snapshot_if_source( + &self, + persistence_id: &str, + sequence_nr: u64, + snapshot: &[u8], + source: &SnapshotSourceFence, + _key_contract: Option<&str>, + ) -> Result<(), PersistenceError> { + self.save_snapshot_atomically(persistence_id, sequence_nr, snapshot, source) .await - .map_err(storage_error)?; - Ok(()) } async fn load_snapshot( @@ -522,280 +867,3 @@ impl EventStore for RedisEventStore { Ok(out) } } - -#[cfg(test)] -mod tests { - use super::*; - use temper_runtime::persistence::EventMetadata; - - fn redis_url() -> Option { - std::env::var("REDIS_URL").ok() - } - - fn test_envelope(event_type: &str, payload: serde_json::Value) -> PersistenceEnvelope { - PersistenceEnvelope { - sequence_nr: 0, - event_type: event_type.to_string(), - payload, - metadata: EventMetadata { - event_id: uuid::Uuid::new_v4(), - causation_id: uuid::Uuid::new_v4(), - correlation_id: uuid::Uuid::new_v4(), - timestamp: chrono::Utc::now(), - actor_id: "redis-test".to_string(), - }, - } - } - - fn unique_persistence_id() -> String { - let id = uuid::Uuid::new_v4(); - format!("test-{id}:Order:ord-{id}") - } - - async fn make_store() -> Option { - let url = redis_url()?; - Some( - RedisEventStore::new(&url) - .await - .expect("failed to connect to Redis"), - ) - } - - #[tokio::test] - async fn append_and_read_events_roundtrip() { - let Some(store) = make_store().await else { - eprintln!("REDIS_URL not set, skipping test"); - return; - }; - let pid = unique_persistence_id(); - - let new_seq = store - .append( - &pid, - 0, - &[ - test_envelope("OrderCreated", serde_json::json!({ "id": "ord-1" })), - test_envelope("OrderApproved", serde_json::json!({ "approved": true })), - ], - ) - .await - .unwrap(); - - assert_eq!(new_seq, 2); - - // Read all events - let events = store.read_events(&pid, 0).await.unwrap(); - assert_eq!(events.len(), 2); - assert_eq!(events[0].sequence_nr, 1); - assert_eq!(events[1].sequence_nr, 2); - assert_eq!(events[0].event_type, "OrderCreated"); - assert_eq!(events[1].event_type, "OrderApproved"); - - // Partial read (from_sequence = 1 should skip event 1) - let partial = store.read_events(&pid, 1).await.unwrap(); - assert_eq!(partial.len(), 1); - assert_eq!(partial[0].sequence_nr, 2); - assert_eq!(partial[0].event_type, "OrderApproved"); - } - - #[tokio::test] - async fn append_with_wrong_sequence_fails() { - let Some(store) = make_store().await else { - eprintln!("REDIS_URL not set, skipping test"); - return; - }; - let pid = unique_persistence_id(); - - store - .append( - &pid, - 0, - &[test_envelope( - "OrderCreated", - serde_json::json!({ "id": "ord-1" }), - )], - ) - .await - .unwrap(); - - let err = store - .append( - &pid, - 0, // stale: actual is 1 - &[test_envelope( - "OrderUpdated", - serde_json::json!({ "step": 2 }), - )], - ) - .await - .unwrap_err(); - - assert!(matches!( - err, - PersistenceError::ConcurrencyViolation { - expected: 0, - actual: 1 - } - )); - } - - #[tokio::test] - async fn snapshot_save_and_load_roundtrip() { - let Some(store) = make_store().await else { - eprintln!("REDIS_URL not set, skipping test"); - return; - }; - let pid = unique_persistence_id(); - - store - .save_snapshot(&pid, 5, b"{\"status\":\"created\"}") - .await - .unwrap(); - - let snapshot = store.load_snapshot(&pid).await.unwrap(); - assert_eq!(snapshot, Some((5, b"{\"status\":\"created\"}".to_vec()))); - - // Overwrite - store - .save_snapshot(&pid, 8, b"{\"status\":\"shipped\"}") - .await - .unwrap(); - - let updated = store.load_snapshot(&pid).await.unwrap(); - assert_eq!(updated, Some((8, b"{\"status\":\"shipped\"}".to_vec()))); - } - - #[tokio::test] - async fn list_entity_ids_returns_distinct_pairs() { - let Some(store) = make_store().await else { - eprintln!("REDIS_URL not set, skipping test"); - return; - }; - let unique = uuid::Uuid::new_v4(); - let tenant_a = format!("tenant-a-{unique}"); - let tenant_b = format!("tenant-b-{unique}"); - - let order_1 = format!("{tenant_a}:Order:ord-1"); - let order_2 = format!("{tenant_a}:Order:ord-2"); - let task_1 = format!("{tenant_a}:Task:task-1"); - let other_tenant = format!("{tenant_b}:Order:ord-9"); - - store - .append( - &order_1, - 0, - &[test_envelope( - "OrderCreated", - serde_json::json!({ "id": "ord-1" }), - )], - ) - .await - .unwrap(); - store - .append( - &order_2, - 0, - &[test_envelope( - "OrderCreated", - serde_json::json!({ "id": "ord-2" }), - )], - ) - .await - .unwrap(); - store - .append( - &task_1, - 0, - &[test_envelope( - "TaskCreated", - serde_json::json!({ "id": "task-1" }), - )], - ) - .await - .unwrap(); - store - .append( - &other_tenant, - 0, - &[test_envelope( - "OrderCreated", - serde_json::json!({ "id": "ord-9" }), - )], - ) - .await - .unwrap(); - - let mut entities = store.list_entity_ids(&tenant_a).await.unwrap(); - entities.sort(); - - assert_eq!( - entities, - vec![ - ("Order".to_string(), "ord-1".to_string()), - ("Order".to_string(), "ord-2".to_string()), - ("Task".to_string(), "task-1".to_string()), - ] - ); - - // Cross-tenant isolation - let other_entities = store.list_entity_ids(&tenant_b).await.unwrap(); - assert_eq!( - other_entities, - vec![("Order".to_string(), "ord-9".to_string())] - ); - } - - #[tokio::test] - async fn concurrent_appends_detect_conflict() { - let Some(store) = make_store().await else { - eprintln!("REDIS_URL not set, skipping test"); - return; - }; - let pid = unique_persistence_id(); - - let store1 = store.clone(); - let store2 = store.clone(); - let pid1 = pid.clone(); - let pid2 = pid.clone(); - - let handle1 = tokio::spawn(async move { - store1 - .append( - &pid1, - 0, - &[test_envelope( - "OrderCreated", - serde_json::json!({ "writer": 1 }), - )], - ) - .await - }); - - let handle2 = tokio::spawn(async move { - store2 - .append( - &pid2, - 0, - &[test_envelope( - "OrderCreated", - serde_json::json!({ "writer": 2 }), - )], - ) - .await - }); - - let (r1, r2) = tokio::join!(handle1, handle2); - let r1 = r1.unwrap(); - let r2 = r2.unwrap(); - - // Exactly one should succeed, the other should get a ConcurrencyViolation. - let successes = [r1.is_ok(), r2.is_ok()].iter().filter(|&&ok| ok).count(); - let conflicts = [&r1, &r2] - .iter() - .filter(|r| matches!(r, Err(PersistenceError::ConcurrencyViolation { .. }))) - .count(); - - assert_eq!(successes, 1, "exactly one writer should succeed"); - assert_eq!(conflicts, 1, "exactly one writer should see a conflict"); - } -} diff --git a/crates/temper-store-redis/src/event_store/atomic.rs b/crates/temper-store-redis/src/event_store/atomic.rs new file mode 100644 index 000000000..d31966f46 --- /dev/null +++ b/crates/temper-store-redis/src/event_store/atomic.rs @@ -0,0 +1,274 @@ +//! Atomic Redis journal and snapshot scripts. + +use super::*; + +impl RedisEventStore { + pub(super) fn encode_snapshot_source( + source: &SnapshotSourceFence, + ) -> Result<(u8, String), PersistenceError> { + match source { + SnapshotSourceFence::Unchecked => Ok((0, String::new())), + SnapshotSourceFence::Absent => Ok((1, String::new())), + SnapshotSourceFence::Exact { sequence_nr, state } => { + let record = SnapshotRecord { + sequence_nr: *sequence_nr, + snapshot: state.clone(), + }; + let encoded = serde_json::to_string(&record) + .map_err(|error| PersistenceError::Serialization(error.to_string()))?; + Ok((2, encoded)) + } + } + } + + pub(super) async fn append_atomically( + &self, + persistence_id: &str, + expected_sequence: u64, + events: &[PersistenceEnvelope], + snapshot_source: &SnapshotSourceFence, + batch_claim: Option<&PersistenceBatchIdempotency>, + ) -> Result<(u64, bool), PersistenceError> { + let (tenant, entity_type, entity_id) = + parse_persistence_id_parts(persistence_id).map_err(PersistenceError::Storage)?; + // A claimed batch must let the atomic Lua path inspect the durable + // claim before optimistic concurrency. Exact retries necessarily carry + // the pre-commit sequence and are stale after the first commit. + if batch_claim.is_none() { + let boundary = self.journal_boundary(persistence_id).await?; + if boundary.latest_sequence != expected_sequence { + return Err(PersistenceError::ConcurrencyViolation { + expected: expected_sequence, + actual: boundary.latest_sequence, + }); + } + } + let seq_key = Self::seq_key(tenant, entity_type, entity_id); + let events_key = Self::events_key(tenant, entity_type, entity_id); + let entities_key = Self::tenant_entities_key(tenant); + let current_segment_key = Self::current_segment_key(tenant, entity_type, entity_id); + let snapshot_key = Self::snapshot_key(tenant, entity_type, entity_id); + let materialization_marker_key = + Self::materialization_marker_key(tenant, entity_type, entity_id); + let terminal_sequence_key = Self::terminal_sequence_key(tenant, entity_type, entity_id); + let batch_idempotency_key = Self::batch_idempotency_key(batch_claim); + + let entity_ref = EntityRef { + entity_type: entity_type.to_string(), + entity_id: entity_id.to_string(), + }; + let entity_ref_json = serde_json::to_string(&entity_ref) + .map_err(|error| PersistenceError::Serialization(error.to_string()))?; + let (snapshot_fence_mode, expected_snapshot) = + Self::encode_snapshot_source(snapshot_source)?; + let retire_snapshot = expected_sequence == 0 + && matches!(snapshot_source, SnapshotSourceFence::Exact { .. }) + && events.first().is_some_and(|event| { + is_state_materialization_event_for(event, entity_type, entity_id) + }); + + let mut encoded_events = Vec::with_capacity(events.len()); + let mut sequence_nr = expected_sequence; + let mut first_terminal_sequence = None; + for event in events { + sequence_nr = sequence_nr.checked_add(1).ok_or_else(|| { + PersistenceError::Storage(format!( + "Redis journal sequence overflow for {persistence_id}" + )) + })?; + let mut envelope = event.clone(); + envelope.sequence_nr = sequence_nr; + if first_terminal_sequence.is_none() && envelope.transitions_to_deleted() { + first_terminal_sequence = Some(sequence_nr); + } + encoded_events.push( + serde_json::to_string(&envelope) + .map_err(|error| PersistenceError::Serialization(error.to_string()))?, + ); + } + + let timestamp = sim_now().to_rfc3339(); + for _attempt in 0..APPEND_POINTER_RETRY_BUDGET { + let current_segment_raw: Option = self + .client + .get(¤t_segment_key) + .await + .map_err(storage_error)?; + let current_segment = match current_segment_raw { + Some(raw) => raw.parse::().map_err(|error| { + PersistenceError::Storage(format!( + "invalid Redis segment pointer '{raw}' for {persistence_id}: {error}" + )) + })?, + None => 0, + }; + let segment_key = Self::segment_key(tenant, entity_type, entity_id, current_segment); + let canonical_segment_key = Self::segment_key(tenant, entity_type, entity_id, 0); + let mut args = Vec::with_capacity(encoded_events.len() + 9); + args.push(expected_sequence.to_string()); + args.push(entity_ref_json.clone()); + args.push(current_segment.to_string()); + args.push(timestamp.clone()); + args.push(snapshot_fence_mode.to_string()); + args.push(expected_snapshot.clone()); + args.push(u8::from(retire_snapshot).to_string()); + args.push(first_terminal_sequence.unwrap_or(0).to_string()); + args.push( + batch_claim + .map(|claim| claim.intent_hash.clone()) + .unwrap_or_default(), + ); + args.extend(encoded_events.iter().cloned()); + + let result: Vec = self + .append_script + .evalsha_with_reload( + &self.client, + vec![ + seq_key.clone(), + events_key.clone(), + entities_key.clone(), + current_segment_key.clone(), + segment_key, + snapshot_key.clone(), + canonical_segment_key, + materialization_marker_key.clone(), + terminal_sequence_key.clone(), + batch_idempotency_key.clone(), + ], + args, + ) + .await + .map_err(storage_error)?; + match result.as_slice() { + [1, new_sequence] => return Ok((*new_sequence as u64, false)), + [0, actual] => { + return Err(PersistenceError::ConcurrencyViolation { + expected: expected_sequence, + actual: *actual as u64, + }); + } + [2, _current_segment] => continue, + [3, _current_sequence] => { + return Err(PersistenceError::SnapshotGenerationChanged); + } + [4, current_sequence] => return Ok((*current_sequence as u64, true)), + [5, _current_sequence] => { + return Err(PersistenceError::Storage(format!( + "atomic batch idempotency key '{}' was reused with a different intent", + batch_claim + .map(|claim| claim.idempotency_key.as_str()) + .unwrap_or_default() + ))); + } + other => { + return Err(PersistenceError::Storage(format!( + "unexpected Redis append script result: {other:?}" + ))); + } + } + } + + Err(PersistenceError::Storage(format!( + "Redis append segment pointer for {persistence_id} did not stabilize after {APPEND_POINTER_RETRY_BUDGET} attempts" + ))) + } + + pub(super) async fn save_snapshot_atomically( + &self, + persistence_id: &str, + sequence_nr: u64, + snapshot: &[u8], + source: &SnapshotSourceFence, + ) -> Result<(), PersistenceError> { + let (tenant, entity_type, entity_id) = + parse_persistence_id_parts(persistence_id).map_err(PersistenceError::Storage)?; + let snapshot_key = Self::snapshot_key(tenant, entity_type, entity_id); + let record = SnapshotRecord { + sequence_nr, + snapshot: snapshot.to_vec(), + }; + let encoded = serde_json::to_string(&record) + .map_err(|error| PersistenceError::Serialization(error.to_string()))?; + + let history_key = Self::snapshot_history_key(tenant, entity_type, entity_id, sequence_nr); + let now = sim_now(); + let history = SnapshotHistoryRecord { + sequence_nr, + snapshot: snapshot.to_vec(), + created_at: now, + }; + let encoded_history = serde_json::to_string(&history) + .map_err(|error| PersistenceError::Serialization(error.to_string()))?; + let (snapshot_fence_mode, expected_snapshot) = Self::encode_snapshot_source(source)?; + + let current_segment_key = Self::current_segment_key(tenant, entity_type, entity_id); + let journal_sequence_key = Self::seq_key(tenant, entity_type, entity_id); + let materialization_marker_key = + Self::materialization_marker_key(tenant, entity_type, entity_id); + let entities_key = Self::tenant_entities_key(tenant); + let entity_ref = serde_json::to_string(&EntityRef { + entity_type: entity_type.to_string(), + entity_id: entity_id.to_string(), + }) + .map_err(|error| PersistenceError::Serialization(error.to_string()))?; + for _attempt in 0..SNAPSHOT_POINTER_RETRY_BUDGET { + let current_segment_raw: Option = self + .client + .get(¤t_segment_key) + .await + .map_err(storage_error)?; + let current_segment = match current_segment_raw { + Some(raw) => raw.parse::().map_err(|error| { + PersistenceError::Storage(format!( + "invalid Redis segment pointer '{raw}' for {persistence_id}: {error}" + )) + })?, + None => 0, + }; + let segment_key = Self::segment_key(tenant, entity_type, entity_id, current_segment); + let next_segment_key = + Self::segment_key(tenant, entity_type, entity_id, current_segment + 1); + let result: i64 = self + .snapshot_script + .evalsha_with_reload( + &self.client, + vec![ + snapshot_key.clone(), + history_key.clone(), + current_segment_key.clone(), + segment_key, + next_segment_key, + journal_sequence_key.clone(), + materialization_marker_key.clone(), + entities_key.clone(), + ], + vec![ + current_segment.to_string(), + sequence_nr.to_string(), + encoded.clone(), + encoded_history.clone(), + now.to_rfc3339(), + snapshot_fence_mode.to_string(), + expected_snapshot.clone(), + entity_ref.clone(), + ], + ) + .await + .map_err(storage_error)?; + match result { + 0..=2 => return Ok(()), + -1 => continue, + -2 => return Err(PersistenceError::SnapshotGenerationChanged), + other => { + return Err(PersistenceError::Storage(format!( + "Redis snapshot script returned unexpected result {other}" + ))); + } + } + } + Err(PersistenceError::Storage(format!( + "Redis snapshot segment pointer for {persistence_id} did not stabilize after {SNAPSHOT_POINTER_RETRY_BUDGET} attempts" + ))) + } +} diff --git a/crates/temper-store-redis/src/event_store/tests/append.rs b/crates/temper-store-redis/src/event_store/tests/append.rs new file mode 100644 index 000000000..e8dd70884 --- /dev/null +++ b/crates/temper-store-redis/src/event_store/tests/append.rs @@ -0,0 +1,353 @@ +//! append scenarios. + +use super::*; + +#[tokio::test] +async fn append_and_read_events_roundtrip() { + let Some(store) = make_store().await else { + eprintln!("REDIS_URL not set, skipping test"); + return; + }; + let pid = unique_persistence_id(); + + let new_seq = store + .append( + &pid, + 0, + &[ + test_envelope("OrderCreated", serde_json::json!({ "id": "ord-1" })), + test_envelope("OrderApproved", serde_json::json!({ "approved": true })), + ], + ) + .await + .unwrap(); + + assert_eq!(new_seq, 2); + + // Read all events + let events = store.read_events(&pid, 0).await.unwrap(); + assert_eq!(events.len(), 2); + assert_eq!(events[0].sequence_nr, 1); + assert_eq!(events[1].sequence_nr, 2); + assert_eq!(events[0].event_type, "OrderCreated"); + assert_eq!(events[1].event_type, "OrderApproved"); + + // Partial read (from_sequence = 1 should skip event 1) + let partial = store.read_events(&pid, 1).await.unwrap(); + assert_eq!(partial.len(), 1); + assert_eq!(partial[0].sequence_nr, 2); + assert_eq!(partial[0].event_type, "OrderApproved"); +} + +#[tokio::test] +async fn legacy_terminal_boundary_is_reconstructed_across_bounded_pages() { + let Some(store) = make_store().await else { + eprintln!("REDIS_URL not set, skipping test"); + return; + }; + let pid = unique_persistence_id(); + let terminal_sequence = JOURNAL_BOUNDARY_PAGE_SIZE as u64 + 1; + let latest_sequence = terminal_sequence + 1; + let events = (1..=latest_sequence) + .map(|sequence| { + if sequence == terminal_sequence { + test_envelope( + "Delete", + serde_json::json!({ + "action": "Delete", + "from_status": "Ready", + "to_status": "Deleted", + }), + ) + } else if sequence == latest_sequence { + test_envelope( + "LegacySuffix", + serde_json::json!({ + "action": "LegacySuffix", + "from_status": "Deleted", + "to_status": "Ready", + }), + ) + } else { + test_envelope("OrderUpdated", serde_json::json!({ "sequence": sequence })) + } + }) + .collect::>(); + assert_eq!( + store.append(&pid, 0, &events).await.unwrap(), + latest_sequence + ); + + let (tenant, entity_type, entity_id) = parse_persistence_id_parts(&pid).unwrap(); + let terminal_key = RedisEventStore::terminal_sequence_key(tenant, entity_type, entity_id); + let _: i64 = store.client.del(&terminal_key).await.unwrap(); + + let boundary = store.journal_boundary(&pid).await.unwrap(); + assert_eq!(boundary.latest_sequence, latest_sequence); + assert_eq!(boundary.first_terminal_sequence, Some(terminal_sequence)); + assert_eq!( + store.client.get::(&terminal_key).await.unwrap(), + terminal_sequence.to_string(), + "legacy reconstruction must persist the first terminal boundary" + ); + assert_eq!( + store.journal_boundary(&pid).await.unwrap(), + boundary, + "later boundary reads must use the installed O(1) metadata" + ); +} + +#[tokio::test] +async fn appending_to_a_legacy_stream_preserves_terminal_boundary_migration() { + let Some(store) = make_store().await else { + eprintln!("REDIS_URL not set, skipping test"); + return; + }; + let pid = unique_persistence_id(); + store + .append( + &pid, + 0, + &[ + test_envelope("Create", serde_json::json!({ "to_status": "Ready" })), + test_envelope( + "Delete", + serde_json::json!({ + "action": "Delete", + "from_status": "Ready", + "to_status": "Deleted", + }), + ), + test_envelope( + "LegacySuffix", + serde_json::json!({ + "from_status": "Deleted", + "to_status": "Ready", + }), + ), + ], + ) + .await + .expect("seed stream with a terminal transition"); + + let (tenant, entity_type, entity_id) = parse_persistence_id_parts(&pid).unwrap(); + let terminal_key = RedisEventStore::terminal_sequence_key(tenant, entity_type, entity_id); + let _: i64 = store.client.del(&terminal_key).await.unwrap(); + + store + .append( + &pid, + 3, + &[test_envelope( + "AppendedBeforeMigration", + serde_json::json!({ "to_status": "Ready" }), + )], + ) + .await + .expect("append to legacy stream before its first boundary read"); + + assert_eq!( + store + .journal_boundary(&pid) + .await + .expect("migrate legacy terminal metadata") + .first_terminal_sequence, + Some(2), + "append must leave missing legacy metadata absent so bounded migration can scan the journal" + ); +} + +#[tokio::test] +async fn append_with_wrong_sequence_fails() { + let Some(store) = make_store().await else { + eprintln!("REDIS_URL not set, skipping test"); + return; + }; + let pid = unique_persistence_id(); + + store + .append( + &pid, + 0, + &[test_envelope( + "OrderCreated", + serde_json::json!({ "id": "ord-1" }), + )], + ) + .await + .unwrap(); + + let err = store + .append( + &pid, + 0, // stale: actual is 1 + &[test_envelope( + "OrderUpdated", + serde_json::json!({ "step": 2 }), + )], + ) + .await + .unwrap_err(); + + assert!(matches!( + err, + PersistenceError::ConcurrencyViolation { + expected: 0, + actual: 1 + } + )); +} + +#[tokio::test] +async fn snapshot_save_and_load_roundtrip() { + let Some(store) = make_store().await else { + eprintln!("REDIS_URL not set, skipping test"); + return; + }; + let pid = unique_persistence_id(); + let (tenant, entity_type, entity_id) = parse_persistence_id_parts(&pid).unwrap(); + let snapshot_key = RedisEventStore::snapshot_key(tenant, entity_type, entity_id); + let pointer_key = RedisEventStore::current_segment_key(tenant, entity_type, entity_id); + let segment_zero_key = RedisEventStore::segment_key(tenant, entity_type, entity_id, 0); + let segment_one_key = RedisEventStore::segment_key(tenant, entity_type, entity_id, 1); + let history_five_key = RedisEventStore::snapshot_history_key(tenant, entity_type, entity_id, 5); + + store + .save_snapshot(&pid, 5, b"{\"status\":\"created\"}") + .await + .unwrap(); + + let snapshot = store.load_snapshot(&pid).await.unwrap(); + assert_eq!(snapshot, Some((5, b"{\"status\":\"created\"}".to_vec()))); + + let raw_snapshot: String = store.client.get(&snapshot_key).await.unwrap(); + let raw_history: String = store.client.get(&history_five_key).await.unwrap(); + let pointer: Option = store.client.get(&pointer_key).await.unwrap(); + let segment_zero: Option = store.client.get(&segment_zero_key).await.unwrap(); + let segment_one: Option = store.client.get(&segment_one_key).await.unwrap(); + assert_eq!(pointer, None, "snapshot-only state has no event segment"); + assert_eq!(segment_zero, None); + assert_eq!(segment_one, None); + + store + .save_snapshot(&pid, 5, b"{\"status\":\"created\"}") + .await + .unwrap(); + assert_eq!( + store.client.get::(&snapshot_key).await.unwrap(), + raw_snapshot, + "identical snapshot writes must not replace the current row" + ); + assert_eq!( + store + .client + .get::(&history_five_key) + .await + .unwrap(), + raw_history, + "identical snapshot writes must not churn history" + ); + assert_eq!( + store + .client + .get::, _>(&pointer_key) + .await + .unwrap(), + pointer + ); + assert_eq!( + store + .client + .get::, _>(&segment_zero_key) + .await + .unwrap(), + segment_zero + ); + assert_eq!( + store + .client + .get::, _>(&segment_one_key) + .await + .unwrap(), + segment_one + ); + + store + .save_snapshot(&pid, 5, b"{\"status\":\"approved\"}") + .await + .unwrap(); + assert_eq!( + store.load_snapshot(&pid).await.unwrap(), + Some((5, b"{\"status\":\"approved\"}".to_vec())) + ); + assert_eq!( + store + .client + .get::, _>(&pointer_key) + .await + .unwrap(), + pointer, + "same-sequence byte replacement must not advance the segment pointer" + ); + assert_eq!( + store + .client + .get::, _>(&segment_zero_key) + .await + .unwrap(), + segment_zero + ); + assert_eq!( + store + .client + .get::, _>(&segment_one_key) + .await + .unwrap(), + segment_one + ); + + store + .save_snapshot(&pid, 4, b"{\"status\":\"delayed\"}") + .await + .unwrap(); + assert_eq!( + store.load_snapshot(&pid).await.unwrap(), + Some((5, b"{\"status\":\"approved\"}".to_vec())), + "an older writer must not regress the current snapshot" + ); + let history_four_key = RedisEventStore::snapshot_history_key(tenant, entity_type, entity_id, 4); + assert_eq!( + store + .client + .get::, _>(&history_four_key) + .await + .unwrap(), + None, + "an ignored older write must not create history" + ); + assert_eq!( + store + .client + .get::, _>(&pointer_key) + .await + .unwrap(), + pointer + ); + + store + .save_snapshot(&pid, 8, b"{\"status\":\"shipped\"}") + .await + .unwrap(); + + let updated = store.load_snapshot(&pid).await.unwrap(); + assert_eq!(updated, Some((8, b"{\"status\":\"shipped\"}".to_vec()))); + assert_eq!( + store + .client + .get::, _>(&pointer_key) + .await + .unwrap(), + None, + "a newer snapshot without a journal must not create event segments" + ); +} diff --git a/crates/temper-store-redis/src/event_store/tests/listing.rs b/crates/temper-store-redis/src/event_store/tests/listing.rs new file mode 100644 index 000000000..9c45765cf --- /dev/null +++ b/crates/temper-store-redis/src/event_store/tests/listing.rs @@ -0,0 +1,137 @@ +//! listing scenarios. + +use super::*; + +#[tokio::test] +async fn list_entity_ids_returns_distinct_pairs() { + let Some(store) = make_store().await else { + eprintln!("REDIS_URL not set, skipping test"); + return; + }; + let unique = uuid::Uuid::new_v4(); + let tenant_a = format!("tenant-a-{unique}"); + let tenant_b = format!("tenant-b-{unique}"); + + let order_1 = format!("{tenant_a}:Order:ord-1"); + let order_2 = format!("{tenant_a}:Order:ord-2"); + let task_1 = format!("{tenant_a}:Task:task-1"); + let other_tenant = format!("{tenant_b}:Order:ord-9"); + + store + .append( + &order_1, + 0, + &[test_envelope( + "OrderCreated", + serde_json::json!({ "id": "ord-1" }), + )], + ) + .await + .unwrap(); + store + .append( + &order_2, + 0, + &[test_envelope( + "OrderCreated", + serde_json::json!({ "id": "ord-2" }), + )], + ) + .await + .unwrap(); + store + .append( + &task_1, + 0, + &[test_envelope( + "TaskCreated", + serde_json::json!({ "id": "task-1" }), + )], + ) + .await + .unwrap(); + store + .append( + &other_tenant, + 0, + &[test_envelope( + "OrderCreated", + serde_json::json!({ "id": "ord-9" }), + )], + ) + .await + .unwrap(); + + let mut entities = store.list_entity_ids(&tenant_a).await.unwrap(); + entities.sort(); + + assert_eq!( + entities, + vec![ + ("Order".to_string(), "ord-1".to_string()), + ("Order".to_string(), "ord-2".to_string()), + ("Task".to_string(), "task-1".to_string()), + ] + ); + + // Cross-tenant isolation + let other_entities = store.list_entity_ids(&tenant_b).await.unwrap(); + assert_eq!( + other_entities, + vec![("Order".to_string(), "ord-9".to_string())] + ); +} + +#[tokio::test] +async fn concurrent_appends_detect_conflict() { + let Some(store) = make_store().await else { + eprintln!("REDIS_URL not set, skipping test"); + return; + }; + let pid = unique_persistence_id(); + + let store1 = store.clone(); + let store2 = store.clone(); + let pid1 = pid.clone(); + let pid2 = pid.clone(); + + let handle1 = tokio::spawn(async move { + store1 + .append( + &pid1, + 0, + &[test_envelope( + "OrderCreated", + serde_json::json!({ "writer": 1 }), + )], + ) + .await + }); + + let handle2 = tokio::spawn(async move { + store2 + .append( + &pid2, + 0, + &[test_envelope( + "OrderCreated", + serde_json::json!({ "writer": 2 }), + )], + ) + .await + }); + + let (r1, r2) = tokio::join!(handle1, handle2); + let r1 = r1.unwrap(); + let r2 = r2.unwrap(); + + // Exactly one should succeed, the other should get a ConcurrencyViolation. + let successes = [r1.is_ok(), r2.is_ok()].iter().filter(|&&ok| ok).count(); + let conflicts = [&r1, &r2] + .iter() + .filter(|r| matches!(r, Err(PersistenceError::ConcurrencyViolation { .. }))) + .count(); + + assert_eq!(successes, 1, "exactly one writer should succeed"); + assert_eq!(conflicts, 1, "exactly one writer should see a conflict"); +} diff --git a/crates/temper-store-redis/src/event_store/tests/materialization.rs b/crates/temper-store-redis/src/event_store/tests/materialization.rs new file mode 100644 index 000000000..6c1e2189b --- /dev/null +++ b/crates/temper-store-redis/src/event_store/tests/materialization.rs @@ -0,0 +1,280 @@ +//! materialization scenarios. + +use super::*; + +#[tokio::test] +async fn materialization_handoff_retires_snapshot_and_blocks_delayed_recreation() { + let Some(store) = make_store().await else { + eprintln!("REDIS_URL not set, skipping test"); + return; + }; + let pid = unique_persistence_id(); + let migration_snapshot = b"{\"status\":\"migration\",\"counter\":10}".to_vec(); + store + .save_snapshot(&pid, 5, &migration_snapshot) + .await + .unwrap(); + + let (tenant, entity_type, entity_id) = parse_persistence_id_parts(&pid).unwrap(); + assert!( + store + .list_entity_ids(tenant) + .await + .unwrap() + .contains(&(entity_type.to_string(), entity_id.to_string())), + "a snapshot-only stream must be discoverable before its first journal append" + ); + assert!( + store + .list_entity_ids_by_type(tenant, entity_type) + .await + .unwrap() + .contains(&entity_id.to_string()), + "type-scoped discovery must include a snapshot-only stream" + ); + let snapshot_key = RedisEventStore::snapshot_key(tenant, entity_type, entity_id); + let history_key = RedisEventStore::snapshot_history_key(tenant, entity_type, entity_id, 5); + let history_before: String = store.client.get(&history_key).await.unwrap(); + + let new_sequence = store + .append_with_index_rows( + &pid, + 0, + &[ + test_envelope( + STATE_MATERIALIZATION_EVENT_TYPE, + serde_json::json!({ + "schema": "temper.state-materialization.v1", + "state": { + "entity_type": entity_type, + "entity_id": entity_id, + "status": "Ready", + "item_count": 0, + "counters": {"counter": 10}, + "booleans": {}, + "lists": {}, + "fields": {"Id": entity_id, "Status": "Ready"}, + "events": [], + "total_event_count": 0, + "events_since_snapshot": 0, + "last_snapshot_sequence_nr": 0, + "sequence_nr": 0, + "processed_idempotency_keys": {}, + }, + }), + ), + test_envelope("OrderIncremented", serde_json::json!({"amount": 1})), + ], + &[], + &[], + IndexReconciliation { + snapshot_source: SnapshotSourceFence::Exact { + sequence_nr: 5, + state: migration_snapshot, + }, + ..IndexReconciliation::default() + }, + ) + .await + .unwrap(); + assert_eq!(new_sequence, 2); + assert_eq!(store.load_snapshot(&pid).await.unwrap(), None); + assert_eq!( + store + .client + .get::, _>(&snapshot_key) + .await + .unwrap(), + None, + "the materialized journal atomically retires its migration snapshot" + ); + + store + .save_snapshot(&pid, 5, b"{\"status\":\"delayed\",\"counter\":10}") + .await + .unwrap(); + + assert_eq!( + store.load_snapshot(&pid).await.unwrap(), + None, + "an ahead-of-journal delayed writer cannot recreate a retired migration snapshot" + ); + assert_eq!( + store.client.get::(&history_key).await.unwrap(), + history_before, + "the delayed no-op must not replace snapshot history" + ); + let events = store.read_events(&pid, 0).await.unwrap(); + assert_eq!(events.len(), 2); + assert_eq!(events[0].event_type, STATE_MATERIALIZATION_EVENT_TYPE); + + store + .append( + &pid, + 2, + &[ + test_envelope("OrderIncremented", serde_json::json!({"amount": 1})), + test_envelope("OrderIncremented", serde_json::json!({"amount": 1})), + test_envelope("OrderIncremented", serde_json::json!({"amount": 1})), + test_envelope("OrderIncremented", serde_json::json!({"amount": 1})), + ], + ) + .await + .unwrap(); + let pointer_key = RedisEventStore::current_segment_key(tenant, entity_type, entity_id); + let pointer_before: Option = store.client.get(&pointer_key).await.unwrap(); + let active_segment_key = RedisEventStore::segment_key( + tenant, + entity_type, + entity_id, + pointer_before.as_deref().unwrap_or("0").parse().unwrap(), + ); + let active_segment_before: Option = + store.client.get(&active_segment_key).await.unwrap(); + + store + .save_snapshot( + &pid, + 5, + b"{\"status\":\"delayed-after-catch-up\",\"counter\":10}", + ) + .await + .unwrap(); + + assert_eq!( + store.load_snapshot(&pid).await.unwrap(), + None, + "an unchecked writer stays retired after the journal catches and passes its sequence" + ); + assert_eq!( + store.client.get::(&history_key).await.unwrap(), + history_before + ); + assert_eq!( + store + .client + .get::, _>(&pointer_key) + .await + .unwrap(), + pointer_before, + "the rejected write must not rotate segment topology" + ); + assert_eq!( + store + .client + .get::, _>(&active_segment_key) + .await + .unwrap(), + active_segment_before, + "the rejected write must not rewrite active segment metadata" + ); + let events = store.read_events(&pid, 0).await.unwrap(); + assert_eq!(events.len(), 6); + assert_eq!(events.last().unwrap().sequence_nr, 6); +} + +#[tokio::test] +async fn snapshot_ahead_of_journal_does_not_rotate_and_append_repairs_legacy_topology() { + let Some(store) = make_store().await else { + eprintln!("REDIS_URL not set, skipping test"); + return; + }; + let pid = unique_persistence_id(); + store + .append( + &pid, + 0, + &[ + test_envelope("OrderCreated", serde_json::json!({"step": 1})), + test_envelope("OrderUpdated", serde_json::json!({"step": 2})), + ], + ) + .await + .unwrap(); + let snapshot = b"{\"status\":\"migration-baseline\"}".to_vec(); + store.save_snapshot(&pid, 5, &snapshot).await.unwrap(); + + let (tenant, entity_type, entity_id) = parse_persistence_id_parts(&pid).unwrap(); + let pointer_key = RedisEventStore::current_segment_key(tenant, entity_type, entity_id); + let segment_zero_key = RedisEventStore::segment_key(tenant, entity_type, entity_id, 0); + let segment_one_key = RedisEventStore::segment_key(tenant, entity_type, entity_id, 1); + assert_eq!( + store.client.get::(&pointer_key).await.unwrap(), + "0", + "a snapshot ahead of journal HWM 2 must not rotate to segment 1" + ); + let canonical: String = store.client.get(&segment_zero_key).await.unwrap(); + let canonical: SegmentRecord = serde_json::from_str(&canonical).unwrap(); + assert_eq!(canonical.end_sequence_nr, Some(2)); + assert_eq!(canonical.snapshot_sequence, None); + assert!(canonical.sealed_at.is_none()); + + let timestamp = sim_now().to_rfc3339(); + let legacy_ghost = serde_json::json!({ + "segment_index": 1, + "start_sequence_nr": 6, + "end_sequence_nr": null, + "snapshot_sequence": null, + "event_count": 0, + "sealed_at": null, + "created_at": timestamp, + }); + let _: () = store + .client + .set( + &segment_one_key, + legacy_ghost.to_string(), + None, + None, + false, + ) + .await + .unwrap(); + let _: () = store + .client + .set(&pointer_key, "1", None, None, false) + .await + .unwrap(); + + store + .append_with_index_rows( + &pid, + 2, + &[test_envelope( + "OrderUpdated", + serde_json::json!({"step": 3}), + )], + &[], + &[], + IndexReconciliation { + snapshot_source: SnapshotSourceFence::Exact { + sequence_nr: 5, + state: snapshot, + }, + ..IndexReconciliation::default() + }, + ) + .await + .unwrap(); + + assert_eq!( + store.client.get::(&pointer_key).await.unwrap(), + "0" + ); + let repaired: String = store.client.get(&segment_zero_key).await.unwrap(); + let repaired: SegmentRecord = serde_json::from_str(&repaired).unwrap(); + assert_eq!(repaired.start_sequence_nr, 1); + assert_eq!(repaired.end_sequence_nr, Some(3)); + assert_eq!(repaired.event_count, 3); + assert_eq!(repaired.snapshot_sequence, None); + assert!(repaired.sealed_at.is_none()); + assert_eq!( + store + .client + .get::, _>(&segment_one_key) + .await + .unwrap(), + None, + "append must remove the legacy future-start segment" + ); +} diff --git a/crates/temper-store-redis/src/event_store/tests/mod.rs b/crates/temper-store-redis/src/event_store/tests/mod.rs new file mode 100644 index 000000000..20bd8b6c0 --- /dev/null +++ b/crates/temper-store-redis/src/event_store/tests/mod.rs @@ -0,0 +1,63 @@ +//! Redis event-store regression scenarios. + +use super::*; +use temper_runtime::persistence::EventMetadata; + +#[test] +fn batch_claim_key_is_unambiguous_across_entity_and_idempotency_boundaries() { + let left = PersistenceBatchIdempotency { + persistence_id: "tenant:Entity:a".to_string(), + idempotency_key: "b:c".to_string(), + intent_hash: "left".to_string(), + }; + let right = PersistenceBatchIdempotency { + persistence_id: "tenant:Entity:a:b".to_string(), + idempotency_key: "c".to_string(), + intent_hash: "right".to_string(), + }; + + assert_ne!( + RedisEventStore::batch_idempotency_key(Some(&left)), + RedisEventStore::batch_idempotency_key(Some(&right)), + "claim ownership must bind the persistence id and idempotency key as distinct components" + ); +} + +fn redis_url() -> Option { + std::env::var("REDIS_URL").ok() +} + +fn test_envelope(event_type: &str, payload: serde_json::Value) -> PersistenceEnvelope { + PersistenceEnvelope { + sequence_nr: 0, + event_type: event_type.to_string(), + payload, + metadata: EventMetadata { + event_id: uuid::Uuid::new_v4(), + causation_id: uuid::Uuid::new_v4(), + correlation_id: uuid::Uuid::new_v4(), + timestamp: chrono::Utc::now(), + actor_id: "redis-test".to_string(), + }, + } +} + +fn unique_persistence_id() -> String { + let id = uuid::Uuid::new_v4(); + format!("test-{id}:Order:ord-{id}") +} + +async fn make_store() -> Option { + let url = redis_url()?; + Some( + RedisEventStore::new(&url) + .await + .expect("failed to connect to Redis"), + ) +} + +mod append; +mod listing; +mod materialization; +mod snapshot_segments; +mod source_fence; diff --git a/crates/temper-store-redis/src/event_store/tests/snapshot_segments.rs b/crates/temper-store-redis/src/event_store/tests/snapshot_segments.rs new file mode 100644 index 000000000..b57a04c20 --- /dev/null +++ b/crates/temper-store-redis/src/event_store/tests/snapshot_segments.rs @@ -0,0 +1,269 @@ +//! snapshot segments scenarios. + +use super::*; + +#[tokio::test] +async fn delayed_snapshot_rotation_preserves_the_durable_journal_tail() { + let Some(store) = make_store().await else { + eprintln!("REDIS_URL not set, skipping test"); + return; + }; + let pid = unique_persistence_id(); + let events = (1..=10) + .map(|sequence| test_envelope("OrderUpdated", serde_json::json!({ "sequence": sequence }))) + .collect::>(); + assert_eq!(store.append(&pid, 0, &events).await.unwrap(), 10); + + store + .save_snapshot(&pid, 5, b"{\"status\":\"halfway\"}") + .await + .unwrap(); + + let (tenant, entity_type, entity_id) = parse_persistence_id_parts(&pid).unwrap(); + let pointer_key = RedisEventStore::current_segment_key(tenant, entity_type, entity_id); + let pointer: String = store.client.get(&pointer_key).await.unwrap(); + assert_eq!(pointer, "1"); + let sealed: String = store + .client + .get(RedisEventStore::segment_key( + tenant, + entity_type, + entity_id, + 0, + )) + .await + .unwrap(); + let sealed: SegmentRecord = serde_json::from_str(&sealed).unwrap(); + assert_eq!(sealed.segment_index, 0); + assert_eq!(sealed.start_sequence_nr, 1); + assert_eq!(sealed.end_sequence_nr, Some(5)); + assert_eq!(sealed.snapshot_sequence, Some(5)); + assert_eq!(sealed.event_count, 5); + assert!(sealed.sealed_at.is_some()); + + let tail: String = store + .client + .get(RedisEventStore::segment_key( + tenant, + entity_type, + entity_id, + 1, + )) + .await + .unwrap(); + let tail: SegmentRecord = serde_json::from_str(&tail).unwrap(); + assert_eq!(tail.segment_index, 1); + assert_eq!(tail.start_sequence_nr, 6); + assert_eq!(tail.end_sequence_nr, Some(10)); + assert_eq!(tail.event_count, 5); + assert!(tail.sealed_at.is_none()); + assert_eq!(sealed.created_at, tail.created_at); + assert_eq!(store.read_events(&pid, 0).await.unwrap().len(), 10); +} + +#[tokio::test] +async fn first_append_after_snapshot_only_state_builds_canonical_event_topology() { + let Some(store) = make_store().await else { + eprintln!("REDIS_URL not set, skipping test"); + return; + }; + let snapshot = b"{\"status\":\"snapshot-only\"}".to_vec(); + let pid = unique_persistence_id(); + store.save_snapshot(&pid, 5, &snapshot).await.unwrap(); + let (tenant, entity_type, entity_id) = parse_persistence_id_parts(&pid).unwrap(); + let pointer_key = RedisEventStore::current_segment_key(tenant, entity_type, entity_id); + let segment_zero_key = RedisEventStore::segment_key(tenant, entity_type, entity_id, 0); + assert_eq!( + store + .client + .get::, _>(&pointer_key) + .await + .unwrap(), + None + ); + assert_eq!( + store + .client + .get::, _>(&segment_zero_key) + .await + .unwrap(), + None + ); + + let sequence_nr = store + .append_with_index_rows( + &pid, + 0, + &[test_envelope( + "OrderUpdated", + serde_json::json!({ "step": 1 }), + )], + &[], + &[], + IndexReconciliation { + snapshot_source: SnapshotSourceFence::Exact { + sequence_nr: 5, + state: snapshot, + }, + ..IndexReconciliation::default() + }, + ) + .await + .unwrap(); + assert_eq!(sequence_nr, 1); + assert_eq!( + store.client.get::(&pointer_key).await.unwrap(), + "0" + ); + let canonical: String = store.client.get(&segment_zero_key).await.unwrap(); + let canonical: SegmentRecord = serde_json::from_str(&canonical).unwrap(); + assert_eq!(canonical.segment_index, 0); + assert_eq!(canonical.start_sequence_nr, 1); + assert_eq!(canonical.end_sequence_nr, Some(1)); + assert_eq!(canonical.snapshot_sequence, None); + assert_eq!(canonical.event_count, 1); + assert!(canonical.sealed_at.is_none()); + + let legacy_pid = unique_persistence_id(); + let legacy_snapshot = b"{\"status\":\"legacy-snapshot-only\"}".to_vec(); + store + .save_snapshot(&legacy_pid, 5, &legacy_snapshot) + .await + .unwrap(); + let (tenant, entity_type, entity_id) = parse_persistence_id_parts(&legacy_pid).unwrap(); + let pointer_key = RedisEventStore::current_segment_key(tenant, entity_type, entity_id); + let segment_zero_key = RedisEventStore::segment_key(tenant, entity_type, entity_id, 0); + let segment_one_key = RedisEventStore::segment_key(tenant, entity_type, entity_id, 1); + let timestamp = sim_now().to_rfc3339(); + let legacy_zero = serde_json::json!({ + "segment_index": 0, + "start_sequence_nr": 1, + "end_sequence_nr": 5, + "snapshot_sequence": 5, + "event_count": 5, + "sealed_at": timestamp, + "created_at": timestamp, + }); + let legacy_one = serde_json::json!({ + "segment_index": 1, + "start_sequence_nr": 6, + "end_sequence_nr": null, + "snapshot_sequence": null, + "event_count": 0, + "sealed_at": null, + "created_at": timestamp, + }); + let _: () = store + .client + .set( + &segment_zero_key, + legacy_zero.to_string(), + None, + None, + false, + ) + .await + .unwrap(); + let _: () = store + .client + .set(&segment_one_key, legacy_one.to_string(), None, None, false) + .await + .unwrap(); + let _: () = store + .client + .set(&pointer_key, "1", None, None, false) + .await + .unwrap(); + + let results = store + .append_batch(&[PersistenceAppend { + persistence_id: legacy_pid.clone(), + expected_sequence: 0, + events: vec![test_envelope( + "OrderUpdated", + serde_json::json!({ "step": 1 }), + )], + key_rows: Vec::new(), + reconcile_keys: false, + key_set_signature: None, + snapshot_source: SnapshotSourceFence::Exact { + sequence_nr: 5, + state: legacy_snapshot, + }, + batch_idempotency: None, + }]) + .await + .unwrap(); + assert_eq!(results[0].sequence_nr, 1); + assert_eq!( + store.client.get::(&pointer_key).await.unwrap(), + "0" + ); + let canonical: String = store.client.get(&segment_zero_key).await.unwrap(); + let canonical: SegmentRecord = serde_json::from_str(&canonical).unwrap(); + assert_eq!(canonical.segment_index, 0); + assert_eq!(canonical.start_sequence_nr, 1); + assert_eq!(canonical.end_sequence_nr, Some(1)); + assert_eq!(canonical.snapshot_sequence, None); + assert_eq!(canonical.event_count, 1); + assert!(canonical.sealed_at.is_none()); + assert_eq!( + store + .client + .get::, _>(&segment_one_key) + .await + .unwrap(), + None, + "legacy snapshot-only active segment must be removed" + ); + assert_eq!(store.read_events(&legacy_pid, 0).await.unwrap().len(), 1); +} + +#[tokio::test] +async fn claimed_batch_retry_checks_content_before_stale_sequence() { + let Some(store) = make_store().await else { + eprintln!("REDIS_URL not set, skipping test"); + return; + }; + let pid = unique_persistence_id(); + let mut append = PersistenceAppend { + persistence_id: pid.clone(), + expected_sequence: 0, + events: vec![test_envelope( + "CompositeEvent", + serde_json::json!({"operation": "first"}), + )], + key_rows: Vec::new(), + reconcile_keys: false, + key_set_signature: None, + snapshot_source: SnapshotSourceFence::Unchecked, + batch_idempotency: Some(PersistenceBatchIdempotency { + persistence_id: pid.clone(), + idempotency_key: "stable-operation".to_string(), + intent_hash: "intent-a".to_string(), + }), + }; + + store + .append_batch(std::slice::from_ref(&append)) + .await + .expect("first claimed append"); + let retry = store + .append_batch(std::slice::from_ref(&append)) + .await + .expect("exact retry must reach the durable claim before stale sequence checks"); + assert!(retry[0].batch_already_applied); + assert_eq!(retry[0].sequence_nr, 1); + assert_eq!(store.read_events(&pid, 0).await.unwrap().len(), 1); + + append + .batch_idempotency + .as_mut() + .expect("batch claim") + .intent_hash = "intent-b".to_string(); + let error = store + .append_batch(&[append]) + .await + .expect_err("same claim key with different content must fail"); + assert!(error.to_string().contains("different intent")); +} diff --git a/crates/temper-store-redis/src/event_store/tests/source_fence.rs b/crates/temper-store-redis/src/event_store/tests/source_fence.rs new file mode 100644 index 000000000..ee1d3e724 --- /dev/null +++ b/crates/temper-store-redis/src/event_store/tests/source_fence.rs @@ -0,0 +1,255 @@ +//! source fence scenarios. + +use super::*; + +#[tokio::test] +async fn checked_snapshot_save_rejects_a_changed_source_without_mutation() { + let Some(store) = make_store().await else { + eprintln!("REDIS_URL not set, skipping test"); + return; + }; + let pid = unique_persistence_id(); + let first = b"{\"status\":\"first\"}".to_vec(); + let replacement = b"{\"status\":\"replacement\"}".to_vec(); + let checked = b"{\"status\":\"checked\"}".to_vec(); + store.save_snapshot(&pid, 5, &first).await.unwrap(); + store.save_snapshot(&pid, 5, &replacement).await.unwrap(); + + let (tenant, entity_type, entity_id) = parse_persistence_id_parts(&pid).unwrap(); + let pointer_key = RedisEventStore::current_segment_key(tenant, entity_type, entity_id); + let segment_zero_key = RedisEventStore::segment_key(tenant, entity_type, entity_id, 0); + let segment_one_key = RedisEventStore::segment_key(tenant, entity_type, entity_id, 1); + let history_key = RedisEventStore::snapshot_history_key(tenant, entity_type, entity_id, 5); + let pointer_before: Option = store.client.get(&pointer_key).await.unwrap(); + let segment_zero_before: Option = store.client.get(&segment_zero_key).await.unwrap(); + let segment_one_before: Option = store.client.get(&segment_one_key).await.unwrap(); + let history_before: String = store.client.get(&history_key).await.unwrap(); + + let stale = store + .save_snapshot_if_source( + &pid, + 5, + &checked, + &SnapshotSourceFence::Exact { + sequence_nr: 5, + state: first, + }, + None, + ) + .await + .unwrap_err(); + assert!(matches!(stale, PersistenceError::SnapshotGenerationChanged)); + let absent = store + .save_snapshot_if_source(&pid, 5, &checked, &SnapshotSourceFence::Absent, None) + .await + .unwrap_err(); + assert!(matches!( + absent, + PersistenceError::SnapshotGenerationChanged + )); + assert_eq!( + store.load_snapshot(&pid).await.unwrap(), + Some((5, replacement.clone())) + ); + assert_eq!( + store.client.get::(&history_key).await.unwrap(), + history_before + ); + assert_eq!( + store + .client + .get::, _>(&pointer_key) + .await + .unwrap(), + pointer_before + ); + assert_eq!( + store + .client + .get::, _>(&segment_zero_key) + .await + .unwrap(), + segment_zero_before + ); + assert_eq!( + store + .client + .get::, _>(&segment_one_key) + .await + .unwrap(), + segment_one_before + ); + + store + .save_snapshot_if_source( + &pid, + 5, + &checked, + &SnapshotSourceFence::Exact { + sequence_nr: 5, + state: replacement, + }, + None, + ) + .await + .unwrap(); + assert_eq!( + store.load_snapshot(&pid).await.unwrap(), + Some((5, checked.clone())) + ); + assert_eq!( + store + .client + .get::, _>(&pointer_key) + .await + .unwrap(), + pointer_before, + "an accepted same-sequence CAS must not rotate" + ); + + store + .save_snapshot_if_source( + &pid, + 8, + b"{\"status\":\"newer\"}", + &SnapshotSourceFence::Exact { + sequence_nr: 5, + state: checked, + }, + None, + ) + .await + .unwrap(); + assert_eq!( + store + .client + .get::, _>(&pointer_key) + .await + .unwrap(), + None, + "an accepted newer CAS without a journal must not rotate" + ); + + let absent_pid = unique_persistence_id(); + store + .save_snapshot_if_source( + &absent_pid, + 1, + b"{\"status\":\"initial\"}", + &SnapshotSourceFence::Absent, + None, + ) + .await + .unwrap(); + assert_eq!( + store.load_snapshot(&absent_pid).await.unwrap(), + Some((1, b"{\"status\":\"initial\"}".to_vec())) + ); +} + +#[tokio::test] +async fn append_rejects_a_changed_snapshot_source_before_mutating_the_journal() { + let Some(store) = make_store().await else { + eprintln!("REDIS_URL not set, skipping test"); + return; + }; + let pid = unique_persistence_id(); + assert_eq!( + store + .append( + &pid, + 0, + &[test_envelope("OrderCreated", serde_json::json!({}))] + ) + .await + .unwrap(), + 1 + ); + let first_snapshot = b"{\"status\":\"created\"}".to_vec(); + let replacement_snapshot = b"{\"status\":\"replaced\"}".to_vec(); + store.save_snapshot(&pid, 1, &first_snapshot).await.unwrap(); + store + .save_snapshot(&pid, 1, &replacement_snapshot) + .await + .unwrap(); + + let (tenant, entity_type, entity_id) = parse_persistence_id_parts(&pid).unwrap(); + let pointer_key = RedisEventStore::current_segment_key(tenant, entity_type, entity_id); + let pointer_before: String = store.client.get(&pointer_key).await.unwrap(); + let active_key = RedisEventStore::segment_key( + tenant, + entity_type, + entity_id, + pointer_before.parse().unwrap(), + ); + let active_before: String = store.client.get(&active_key).await.unwrap(); + let next_event = test_envelope("OrderUpdated", serde_json::json!({ "step": 2 })); + + let stale = store + .append_with_index_rows( + &pid, + 1, + std::slice::from_ref(&next_event), + &[], + &[], + IndexReconciliation { + snapshot_source: SnapshotSourceFence::Exact { + sequence_nr: 1, + state: first_snapshot, + }, + ..IndexReconciliation::default() + }, + ) + .await + .unwrap_err(); + assert!(matches!(stale, PersistenceError::SnapshotGenerationChanged)); + assert_eq!(store.read_events(&pid, 0).await.unwrap().len(), 1); + assert_eq!( + store.client.get::(&pointer_key).await.unwrap(), + pointer_before + ); + assert_eq!( + store.client.get::(&active_key).await.unwrap(), + active_before + ); + + let absent = store + .append_with_index_rows( + &pid, + 1, + std::slice::from_ref(&next_event), + &[], + &[], + IndexReconciliation { + snapshot_source: SnapshotSourceFence::Absent, + ..IndexReconciliation::default() + }, + ) + .await + .unwrap_err(); + assert!(matches!( + absent, + PersistenceError::SnapshotGenerationChanged + )); + assert_eq!(store.read_events(&pid, 0).await.unwrap().len(), 1); + + let new_sequence = store + .append_with_index_rows( + &pid, + 1, + &[next_event], + &[], + &[], + IndexReconciliation { + snapshot_source: SnapshotSourceFence::Exact { + sequence_nr: 1, + state: replacement_snapshot, + }, + ..IndexReconciliation::default() + }, + ) + .await + .unwrap(); + assert_eq!(new_sequence, 2); + assert_eq!(store.read_events(&pid, 0).await.unwrap().len(), 2); +} diff --git a/crates/temper-store-sim/src/append.rs b/crates/temper-store-sim/src/append.rs new file mode 100644 index 000000000..c6f0aea0e --- /dev/null +++ b/crates/temper-store-sim/src/append.rs @@ -0,0 +1,261 @@ +//! Source-fenced deterministic single-stream appends. + +use super::*; +use crate::source_fence::{ + append_retires_snapshot, snapshot_source_matches, update_segments_after_append_locked, +}; + +impl SimEventStore { + pub(super) async fn append_with_index_rows_inner( + &self, + persistence_id: &str, + expected_sequence: u64, + events: &[PersistenceEnvelope], + key_rows: &[temper_runtime::persistence::EntityKeyRow], + vector_rows: &[EntityVectorRow], + reconciliation: IndexReconciliation, + ) -> Result { + let (postcommit_pause, append_pause) = { + let mut inner = self + .inner + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let postcommit_pause = inner + .pending_postcommit_append_pauses + .get_mut(persistence_id) + .and_then(VecDeque::pop_front); + if inner + .pending_postcommit_append_pauses + .get(persistence_id) + .is_some_and(VecDeque::is_empty) + { + inner + .pending_postcommit_append_pauses + .remove(persistence_id); + } + let pause = inner + .pending_append_pauses + .get_mut(persistence_id) + .and_then(VecDeque::pop_front); + if inner + .pending_append_pauses + .get(persistence_id) + .is_some_and(VecDeque::is_empty) + { + inner.pending_append_pauses.remove(persistence_id); + } + (postcommit_pause, pause) + }; + if let Some(pause) = append_pause { + pause.reached.notify_one(); + pause.resume.notified().await; + } + let new_seq = { + let mut inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock + + // Deterministic one-shot injection (see `inject_concurrency_violations`). + // Consumes one counter per call; falls back to normal flow once drained. + // + // The reported `actual` equals `expected_sequence` — the journal has + // not actually moved, so an authoritative replay will land back at + // `expected_sequence`. Any code that asserts + // `post_replay_sequence >= actual` still holds without this injection + // lying about journal state. + let pending_cv = inner + .pending_concurrency_violations + .get(persistence_id) + .copied() + .unwrap_or(0); + if pending_cv > 0 { + if pending_cv == 1 { + inner.pending_concurrency_violations.remove(persistence_id); + } else { + inner + .pending_concurrency_violations + .insert(persistence_id.to_string(), pending_cv - 1); + } + return Err(PersistenceError::ConcurrencyViolation { + expected: expected_sequence, + actual: expected_sequence, + }); + } + + // Fault injection: spurious concurrency violation (probabilistic). + let cv_prob = inner.faults.concurrency_violation_prob; + if inner.rng.chance(cv_prob) { + return Err(PersistenceError::ConcurrencyViolation { + expected: expected_sequence, + actual: expected_sequence.wrapping_add(1), + }); + } + + // Fault injection: write failure. + let wf_prob = inner.faults.write_failure_prob; + if inner.rng.chance(wf_prob) { + return Err(PersistenceError::Storage( + "SimEventStore: injected write failure".into(), + )); + } + + // Check optimistic concurrency. + let current_seq = inner + .journals + .get(persistence_id) + .and_then(|journal| journal.last().map(|e| e.sequence_nr)) + .unwrap_or(0); + if current_seq != expected_sequence { + return Err(PersistenceError::ConcurrencyViolation { + expected: expected_sequence, + actual: current_seq, + }); + } + if !snapshot_source_matches( + inner.snapshots.get(persistence_id), + &reconciliation.snapshot_source, + ) { + return Err(PersistenceError::SnapshotGenerationChanged); + } + + // Parse once before any journal mutation. This keeps key/vector ownership + // aligned with the runtime's canonical persistence-id parser, including the + // supported legacy two-segment form, and makes a malformed ID fail atomically. + let index_identity = + parse_persistence_id_parts(persistence_id).map_err(PersistenceError::Storage)?; + + // ADR-0153: validate declared-key uniqueness BEFORE writing the journal, so + // a reject is atomic — the journal must not advance on a rejected co-commit. + // A *different* entity already holding the key is the violation. + if reconciliation.keys { + let (tenant, entity_type, entity_id) = index_identity; + for row in key_rows { + if let Some(existing) = inner.key_index.get(&( + tenant.to_string(), + entity_type.to_string(), + row.key_name.clone(), + row.key_hash.clone(), + )) && existing.0.as_str() != entity_id + { + return Err(PersistenceError::Storage(format!( + "duplicate declared key '{}' for {entity_type}: held by {}", + row.key_name, existing.0, + ))); + } + } + } + + if reconciliation.keys { + let (tenant, entity_type, _) = index_identity; + reconcile_key_contract_locked( + &mut inner, + tenant, + entity_type, + reconciliation.key_set_signature.as_deref(), + KeyContractUse::LiveWrite, + )?; + } else if !events.is_empty() { + let (tenant, entity_type, _) = index_identity; + invalidate_coverage_for_unreconciled_append_locked( + &mut inner, + tenant, + entity_type, + )?; + } + + let mut new_seq = expected_sequence; + let mut stored_events = Vec::with_capacity(events.len()); + for event in events { + new_seq += 1; + // Store with correct sequence number (ignore the one in the envelope, + // use monotonic counter like the real stores do). + let mut stored = event.clone(); + stored.sequence_nr = new_seq; + stored_events.push(stored); + } + inner + .journals + .entry(persistence_id.to_string()) + .or_default() + .extend(stored_events); + + update_segments_after_append_locked( + &mut inner, + persistence_id, + expected_sequence, + new_seq, + ); + + // ADR-0153/0171: co-commit the entity's EXACT declared key set under the + // SAME lock as the journal write above (uniqueness was validated before the + // journal, so this only mutates — never fails). Empty rows release every + // prior claim, including rows for declarations that no longer exist. + if reconciliation.keys { + let (tenant, entity_type, entity_id) = index_identity; + inner.key_index.retain(|(t, et, _, _), (eid, _)| { + !(t.as_str() == tenant + && et.as_str() == entity_type + && eid.as_str() == entity_id) + }); + for row in key_rows { + inner.key_index.insert( + ( + tenant.to_string(), + entity_type.to_string(), + row.key_name.clone(), + row.key_hash.clone(), + ), + (entity_id.to_string(), new_seq), + ); + } + } + + // ADR-0155: co-commit the derived vector-index rows under the SAME lock as + // the journal write. When the entity's type declares vector paths + // (`reconciliation.vectors`), DELETE all of the entity's rows first, then insert + // the current ones — so a delete transition or a cleared vector/model + // property (empty `vector_rows`) purges the stale rows instead of leaving + // them to rank forever. No uniqueness constraint — vectors are derived state. + if reconciliation.vectors { + let (tenant, entity_type, entity_id) = index_identity; + inner.vector_index.retain(|(t, et, _, _, eid), _| { + !(t.as_str() == tenant + && et.as_str() == entity_type + && eid.as_str() == entity_id) + }); + for row in vector_rows { + inner.vector_index.insert( + ( + tenant.to_string(), + entity_type.to_string(), + row.decl_name.clone(), + row.model_tag.clone(), + entity_id.to_string(), + ), + row.vector.clone(), + ); + } + } + + if append_retires_snapshot( + expected_sequence, + events, + &reconciliation.snapshot_source, + index_identity.1, + index_identity.2, + ) { + inner.snapshots.remove(persistence_id); + } + + new_seq + }; + + // Model a lost/delayed acknowledgement after durability through an + // explicit deterministic handshake. The store lock is already released, + // so tests can prove retry behavior without ambient wall-clock races. + if let Some(pause) = postcommit_pause { + pause.reached.notify_one(); + pause.resume.notified().await; + } + + Ok(new_seq) + } +} diff --git a/crates/temper-store-sim/src/append_batch.rs b/crates/temper-store-sim/src/append_batch.rs new file mode 100644 index 000000000..6c71e1377 --- /dev/null +++ b/crates/temper-store-sim/src/append_batch.rs @@ -0,0 +1,278 @@ +//! Atomic deterministic multi-stream appends. + +use super::*; +use crate::source_fence::{ + append_retires_snapshot, snapshot_source_matches, update_segments_after_append_locked, +}; + +impl SimEventStore { + pub(super) async fn append_batch_inner( + &self, + appends: &[PersistenceAppend], + ) -> Result, PersistenceError> { + if appends.is_empty() { + return Ok(Vec::new()); + } + + let pause = self + .inner + .lock() + .expect("SimEventStore lock poisoned") + .pending_batch_pauses + .pop_front(); + if let Some(pause) = pause { + pause.reached.notify_one(); + pause.resume.notified().await; + } + + let mut inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock + + let mut seen = std::collections::BTreeSet::new(); + let mut type_contracts = BTreeMap::new(); + let mut unreconciled_types = BTreeSet::new(); + let mut batch_claim = None; + for append in appends { + if !append.reconcile_keys && !append.key_rows.is_empty() { + return Err(PersistenceError::Storage(format!( + "SimEventStore: append_batch stream '{}' supplied key rows without exact reconciliation", + append.persistence_id + ))); + } + if !seen.insert(append.persistence_id.as_str()) { + return Err(PersistenceError::Storage(format!( + "SimEventStore: duplicate persistence_id '{}' in append_batch", + append.persistence_id + ))); + } + if let Some(claim) = &append.batch_idempotency + && batch_claim.replace(claim).is_some() + { + return Err(PersistenceError::Storage( + "SimEventStore: append_batch supplied more than one idempotency claim" + .to_string(), + )); + } + let (tenant, entity_type, _) = parse_persistence_id_parts(&append.persistence_id) + .map_err(PersistenceError::Storage)?; + if append.reconcile_keys { + let type_key = (tenant.to_string(), entity_type.to_string()); + if let Some(existing) = type_contracts.get(&type_key) { + if existing != &append.key_set_signature { + return Err(PersistenceError::Storage(format!( + "SimEventStore: append_batch supplied inconsistent key contracts for {tenant}:{entity_type}" + ))); + } + } else { + type_contracts.insert(type_key, append.key_set_signature.clone()); + } + } else if !append.events.is_empty() { + unreconciled_types.insert((tenant.to_string(), entity_type.to_string())); + } + } + + if let Some(claim) = batch_claim { + let claim_key = (claim.persistence_id.clone(), claim.idempotency_key.clone()); + if let Some(committed_hash) = inner.batch_idempotency.get(&claim_key) { + if committed_hash != &claim.intent_hash { + return Err(PersistenceError::Storage(format!( + "atomic batch idempotency key '{}' was reused with a different intent", + claim.idempotency_key + ))); + } + return Ok(appends + .iter() + .map(|append| PersistenceAppendResult { + persistence_id: append.persistence_id.clone(), + sequence_nr: inner + .journals + .get(&append.persistence_id) + .and_then(|journal| journal.last()) + .map(|event| event.sequence_nr) + .unwrap_or(0), + batch_already_applied: true, + }) + .collect()); + } + } + + for append in appends { + let pending_cv = inner + .pending_concurrency_violations + .get(&append.persistence_id) + .copied() + .unwrap_or(0); + if pending_cv > 0 { + if pending_cv == 1 { + inner + .pending_concurrency_violations + .remove(&append.persistence_id); + } else { + inner + .pending_concurrency_violations + .insert(append.persistence_id.clone(), pending_cv - 1); + } + return Err(PersistenceError::ConcurrencyViolation { + expected: append.expected_sequence, + actual: append.expected_sequence, + }); + } + } + + // Fault injection happens before mutation so a batch either writes + // every stream or no stream. + let cv_prob = inner.faults.concurrency_violation_prob; + if inner.rng.chance(cv_prob) { + let first = &appends[0]; + return Err(PersistenceError::ConcurrencyViolation { + expected: first.expected_sequence, + actual: first.expected_sequence.wrapping_add(1), + }); + } + let wf_prob = inner.faults.write_failure_prob; + if inner.rng.chance(wf_prob) { + return Err(PersistenceError::Storage( + "SimEventStore: injected batch write failure".into(), + )); + } + + for append in appends { + let current_seq = inner + .journals + .get(&append.persistence_id) + .and_then(|journal| journal.last()) + .map(|event| event.sequence_nr) + .unwrap_or(0); + if current_seq != append.expected_sequence { + return Err(PersistenceError::ConcurrencyViolation { + expected: append.expected_sequence, + actual: current_seq, + }); + } + if !snapshot_source_matches( + inner.snapshots.get(&append.persistence_id), + &append.snapshot_source, + ) { + return Err(PersistenceError::SnapshotGenerationChanged); + } + } + + // Build the complete post-batch key map before mutating any journal. Removing + // every participating entity first permits an atomic ownership transfer; a + // conflicting final claim rejects the whole batch with both journals and the + // live key map untouched. + let mut next_key_index = inner.key_index.clone(); + for append in appends.iter().filter(|append| append.reconcile_keys) { + let (tenant, entity_type, entity_id) = + parse_persistence_id_parts(&append.persistence_id) + .map_err(PersistenceError::Storage)?; + next_key_index.retain(|(t, et, _, _), (owner, _)| { + !(t.as_str() == tenant && et.as_str() == entity_type && owner.as_str() == entity_id) + }); + } + for append in appends.iter().filter(|append| append.reconcile_keys) { + let (tenant, entity_type, entity_id) = + parse_persistence_id_parts(&append.persistence_id) + .map_err(PersistenceError::Storage)?; + for row in &append.key_rows { + let slot = ( + tenant.to_string(), + entity_type.to_string(), + row.key_name.clone(), + row.key_hash.clone(), + ); + if let Some(existing) = next_key_index.get(&slot) + && existing.0.as_str() != entity_id + { + return Err(PersistenceError::Storage(format!( + "duplicate declared key '{}' for {entity_type}: held by {}", + row.key_name, existing.0 + ))); + } + let final_sequence = append + .expected_sequence + .checked_add(append.events.len() as u64) + .ok_or_else(|| { + PersistenceError::Storage(format!( + "append_batch sequence overflow for '{}'", + append.persistence_id + )) + })?; + next_key_index.insert(slot, (entity_id.to_string(), final_sequence)); + } + } + + let contract_before = inner.key_index_contract.clone(); + let watermark_before = inner.key_index_watermark.clone(); + for ((tenant, entity_type), signature) in &type_contracts { + if let Err(error) = reconcile_key_contract_locked( + &mut inner, + tenant, + entity_type, + signature.as_deref(), + KeyContractUse::LiveWrite, + ) { + inner.key_index_contract = contract_before; + inner.key_index_watermark = watermark_before; + return Err(error); + } + } + for (tenant, entity_type) in &unreconciled_types { + if let Err(error) = + invalidate_coverage_for_unreconciled_append_locked(&mut inner, tenant, entity_type) + { + inner.key_index_contract = contract_before; + inner.key_index_watermark = watermark_before; + return Err(error); + } + } + + let mut results = Vec::with_capacity(appends.len()); + for append in appends { + let mut new_seq = append.expected_sequence; + { + let journal = inner + .journals + .entry(append.persistence_id.clone()) + .or_default(); + for event in &append.events { + new_seq += 1; + let mut stored = event.clone(); + stored.sequence_nr = new_seq; + journal.push(stored); + } + } + update_segments_after_append_locked( + &mut inner, + &append.persistence_id, + append.expected_sequence, + new_seq, + ); + let (_, entity_type, entity_id) = parse_persistence_id_parts(&append.persistence_id) + .map_err(PersistenceError::Storage)?; + if append_retires_snapshot( + append.expected_sequence, + &append.events, + &append.snapshot_source, + entity_type, + entity_id, + ) { + inner.snapshots.remove(&append.persistence_id); + } + results.push(PersistenceAppendResult { + persistence_id: append.persistence_id.clone(), + sequence_nr: new_seq, + batch_already_applied: false, + }); + } + + inner.key_index = next_key_index; + if let Some(claim) = batch_claim { + inner.batch_idempotency.insert( + (claim.persistence_id.clone(), claim.idempotency_key.clone()), + claim.intent_hash.clone(), + ); + } + + Ok(results) + } +} diff --git a/crates/temper-store-sim/src/fault_injection.rs b/crates/temper-store-sim/src/fault_injection.rs new file mode 100644 index 000000000..4886687c4 --- /dev/null +++ b/crates/temper-store-sim/src/fault_injection.rs @@ -0,0 +1,113 @@ +//! Deterministic fault configuration, RNG, and append barriers. + +use std::sync::Arc; +use tokio::sync::Notify; + +#[derive(Clone, Debug)] +pub(super) struct SimAppendPauseState { + pub(super) reached: Arc, + pub(super) resume: Arc, +} + +/// Deterministic test barrier that pauses one append at an injected boundary, +/// allowing a test to advance another simulated state transition. +#[derive(Clone, Debug)] +pub struct SimAppendPause { + pub(super) state: SimAppendPauseState, +} + +impl SimAppendPause { + /// Wait until the target append reaches the injected barrier. + pub async fn wait_until_reached(&self) { + self.state.reached.notified().await; + } + + /// Release the paused append. + pub fn resume(&self) { + self.state.resume.notify_one(); + } +} + +/// Fault injection configuration for simulation. +/// +/// Controls the probability of injected failures during event store operations. +/// All probabilities are in \[0.0, 1.0\]. +#[derive(Debug, Clone)] +pub struct SimFaultConfig { + /// Probability of a write failure on `append()`. + pub write_failure_prob: f64, + /// Probability of a spurious concurrency violation on `append()`. + pub concurrency_violation_prob: f64, + /// Probability of truncating journal on `read_events()`. + pub read_truncation_prob: f64, + /// Probability of snapshot save failure. + pub snapshot_failure_prob: f64, +} + +impl SimFaultConfig { + /// No fault injection — all operations succeed. + pub fn none() -> Self { + Self { + write_failure_prob: 0.0, + concurrency_violation_prob: 0.0, + read_truncation_prob: 0.0, + snapshot_failure_prob: 0.0, + } + } + + /// Heavy fault injection for stress testing. + pub fn heavy() -> Self { + Self { + write_failure_prob: 0.05, + concurrency_violation_prob: 0.02, + read_truncation_prob: 0.01, + snapshot_failure_prob: 0.03, + } + } +} + +impl Default for SimFaultConfig { + fn default() -> Self { + Self::none() + } +} + +/// Deterministic pseudo-random number generator for fault injection. +/// +/// Simple xorshift64 — fast, deterministic, good enough for fault injection. +/// Uses `BTreeMap` internally (DST compliance: deterministic iteration order). +#[derive(Debug, Clone)] +pub struct DeterministicRng { + state: u64, +} + +impl DeterministicRng { + /// Create a new RNG with the given seed. + pub fn new(seed: u64) -> Self { + Self { + state: if seed == 0 { 1 } else { seed }, + } + } + + /// Generate next u64. + pub fn next_u64(&mut self) -> u64 { + let mut x = self.state; + x ^= x << 13; + x ^= x >> 7; + x ^= x << 17; + self.state = x; + x + } + + /// Return true with the given probability \[0.0, 1.0\]. + pub fn chance(&mut self, prob: f64) -> bool { + if prob <= 0.0 { + return false; + } + if prob >= 1.0 { + return true; + } + let threshold = (prob * u64::MAX as f64) as u64; + self.next_u64() < threshold + } +} diff --git a/crates/temper-store-sim/src/key_index.rs b/crates/temper-store-sim/src/key_index.rs new file mode 100644 index 000000000..a69b3b4ba --- /dev/null +++ b/crates/temper-store-sim/src/key_index.rs @@ -0,0 +1,492 @@ +//! Deterministic key-ownership reconciliation and coverage fencing. + +use std::collections::BTreeSet; + +use temper_runtime::persistence::{ + EntityKeyLookup, EntityKeyRow, KeyIndexBackfillFence, PersistenceEnvelope, PersistenceError, + decode_activated_key_contract, encode_activated_key_contract, +}; +use temper_runtime::tenant::parse_persistence_id_parts; + +use super::SimEventStoreInner; + +const UNKNOWN_KEY_SET_SIGNATURE: &str = ""; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) enum KeyContractUse { + LiveWrite, + Backfill, +} + +pub(super) fn reconcile_key_contract_locked( + inner: &mut SimEventStoreInner, + tenant: &str, + entity_type: &str, + key_set_signature: Option<&str>, + contract_use: KeyContractUse, +) -> Result { + let type_key = (tenant.to_string(), entity_type.to_string()); + let raw_contract = key_set_signature.unwrap_or(UNKNOWN_KEY_SET_SIGNATURE); + let (supplied, attempted_epoch) = decode_activated_key_contract(raw_contract); + let supplied = supplied.to_string(); + match ( + contract_use, + inner.key_index_activated_contract.get(&type_key), + ) { + (KeyContractUse::LiveWrite | KeyContractUse::Backfill, Some((active, _, _))) + if active != &supplied => + { + return Err(PersistenceError::KeyContractNotActive { + activated_signature: active.clone(), + attempted_signature: supplied, + }); + } + (KeyContractUse::LiveWrite, Some((_, activated_epoch, _))) + if attempted_epoch != Some(*activated_epoch) => + { + return Err(PersistenceError::KeyContractActivationStale { + activated_epoch: *activated_epoch, + attempted_epoch, + }); + } + (KeyContractUse::LiveWrite, Some((active, activated_epoch, _))) + if inner + .key_index_watermark + .get(&type_key) + .is_none_or(|covered| covered != active) => + { + return Err(PersistenceError::KeyContractActivationNotReady { + activated_epoch: *activated_epoch, + activated_signature: active.clone(), + }); + } + // Legacy tables and backfill passes do not establish an activation + // epoch. Epoch enforcement starts only when the runtime explicitly + // activates a durable spec contract. + _ => {} + } + match inner.key_index_contract.get(&type_key).cloned() { + Some((current, revision)) if current == supplied => Ok(revision), + Some((_, revision)) => { + let next = revision.checked_add(1).ok_or_else(|| { + PersistenceError::Storage(format!( + "SimEventStore: key contract revision overflow for {tenant}:{entity_type}" + )) + })?; + inner + .key_index_contract + .insert(type_key.clone(), (supplied, next)); + inner.key_index_watermark.remove(&type_key); + Ok(next) + } + None => { + inner + .key_index_contract + .insert(type_key.clone(), (supplied, 1)); + inner.key_index_watermark.remove(&type_key); + Ok(1) + } + } +} + +pub(super) fn activate_key_contract_locked( + inner: &mut SimEventStoreInner, + tenant: &str, + entity_type: &str, + key_set_signature: &str, + spec_fingerprint: &str, + purge_existing_rows: bool, +) -> Result { + let (key_set_signature, _) = decode_activated_key_contract(key_set_signature); + let type_key = (tenant.to_string(), entity_type.to_string()); + let activation_epoch = inner + .key_index_activated_contract + .get(&type_key) + .map(|(_, epoch, _)| *epoch) + .unwrap_or(0) + .checked_add(1) + .ok_or_else(|| { + PersistenceError::Storage(format!( + "SimEventStore: key activation epoch overflow for {tenant}:{entity_type}" + )) + })?; + let prior_activation = inner.key_index_activated_contract.get(&type_key).cloned(); + let semantic_contract_changed = + prior_activation + .as_ref() + .is_none_or(|(active_key_set, _, active_fingerprint)| { + active_key_set != key_set_signature || active_fingerprint != spec_fingerprint + }); + let current_key_set = inner.key_index_contract.get(&type_key).cloned(); + inner.key_index_activated_contract.insert( + type_key.clone(), + ( + key_set_signature.to_string(), + activation_epoch, + spec_fingerprint.to_string(), + ), + ); + let activated_contract = encode_activated_key_contract(key_set_signature, activation_epoch); + reconcile_key_contract_locked( + inner, + tenant, + entity_type, + Some(&activated_contract), + KeyContractUse::Backfill, + )?; + if semantic_contract_changed + && current_key_set + .as_ref() + .is_some_and(|(signature, _)| signature == key_set_signature) + { + let (_, revision) = inner + .key_index_contract + .get(&type_key) + .cloned() + .expect("activation reconciliation installed a contract"); + let next = revision.checked_add(1).ok_or_else(|| { + PersistenceError::Storage(format!( + "SimEventStore: key contract revision overflow for {tenant}:{entity_type}" + )) + })?; + inner + .key_index_contract + .insert(type_key.clone(), (key_set_signature.to_string(), next)); + inner.key_index_watermark.remove(&type_key); + } + if purge_existing_rows || semantic_contract_changed { + inner.key_index.retain(|(row_tenant, row_type, _, _), _| { + row_tenant != tenant || row_type != entity_type + }); + } + Ok(activation_epoch) +} + +/// Invalidate an in-flight coverage proof when a snapshot mutation changes the +/// durable state used to derive ownership. +/// +/// Even at or below the journal high-water, snapshot bytes are the recovery +/// baseline for legacy fields. Keeping the current signature while advancing its +/// revision makes publication ABA-safe without turning a snapshot change into a +/// contract change. Identical snapshot writes are filtered by the caller. +pub(super) fn invalidate_coverage_for_snapshot_write_locked( + inner: &mut SimEventStoreInner, + persistence_id: &str, +) -> Result<(), PersistenceError> { + let (tenant, entity_type, _) = + parse_persistence_id_parts(persistence_id).map_err(PersistenceError::Storage)?; + let type_key = (tenant.to_string(), entity_type.to_string()); + let Some((signature, revision)) = inner.key_index_contract.get(&type_key).cloned() else { + return Ok(()); + }; + let next = revision.checked_add(1).ok_or_else(|| { + PersistenceError::Storage(format!( + "SimEventStore: key reconciliation revision overflow for {tenant}:{entity_type}" + )) + })?; + inner + .key_index_contract + .insert(type_key.clone(), (signature, next)); + inner.key_index_watermark.remove(&type_key); + Ok(()) +} + +/// Close an ownership proof before a journal append that does not carry an +/// exact declared-key reconciliation. The revision change makes a racing +/// repair publication lose its compare-and-set after the append commits. +pub(super) fn invalidate_coverage_for_unreconciled_append_locked( + inner: &mut SimEventStoreInner, + tenant: &str, + entity_type: &str, +) -> Result<(), PersistenceError> { + let type_key = (tenant.to_string(), entity_type.to_string()); + let Some((signature, revision)) = inner.key_index_contract.get(&type_key).cloned() else { + return Ok(()); + }; + let next = revision.checked_add(1).ok_or_else(|| { + PersistenceError::Storage(format!( + "SimEventStore: key reconciliation revision overflow for {tenant}:{entity_type}" + )) + })?; + inner + .key_index_contract + .insert(type_key.clone(), (signature, next)); + inner.key_index_watermark.remove(&type_key); + Ok(()) +} + +pub(super) fn backfill_entity_keys( + inner: &mut SimEventStoreInner, + tenant: &str, + entity_type: &str, + entity_id: &str, + expected_sequence: u64, + contract_fence: KeyIndexBackfillFence<'_>, + key_rows: &[EntityKeyRow], +) -> Result<(), PersistenceError> { + let type_key = (tenant.to_string(), entity_type.to_string()); + let current_contract = inner.key_index_contract.get(&type_key).cloned(); + let actual_revision = current_contract + .as_ref() + .map(|(_, revision)| *revision) + .unwrap_or(0); + if !matches!( + current_contract, + Some((ref signature, revision)) + if signature == contract_fence.key_set_signature + && revision == contract_fence.contract_revision + ) { + return Err(PersistenceError::KeyContractChanged { + expected_signature: contract_fence.key_set_signature.to_string(), + expected_revision: contract_fence.contract_revision, + actual_signature: current_contract.map(|(signature, _)| signature), + actual_revision, + }); + } + + let persistence_id = format!("{tenant}:{entity_type}:{entity_id}"); + let snapshot_matches = match ( + contract_fence.expected_snapshot, + inner.snapshots.get(&persistence_id), + ) { + (None, None) => true, + (Some(expected), Some((sequence_nr, state))) => { + *sequence_nr == expected.sequence_nr && state.as_slice() == expected.state + } + _ => false, + }; + if !snapshot_matches { + return Err(PersistenceError::SnapshotGenerationChanged); + } + let journal_sequence = inner + .journals + .get(&persistence_id) + .and_then(|journal| journal.last()) + .map(|event| event.sequence_nr) + .unwrap_or(0); + let snapshot_sequence = inner + .snapshots + .get(&persistence_id) + .map(|(sequence_nr, _)| *sequence_nr) + .unwrap_or(0); + let current_sequence = if journal_sequence > 0 { + journal_sequence + } else { + snapshot_sequence + }; + if journal_sequence != contract_fence.expected_journal_sequence { + return Err(PersistenceError::JournalBoundaryChanged { + expected: contract_fence.expected_journal_sequence, + actual: journal_sequence, + }); + } + let current_entity_live = entity_is_live(inner, &persistence_id); + if current_entity_live != contract_fence.expected_entity_live { + return Err(PersistenceError::EntityLivenessChanged { + expected_live: contract_fence.expected_entity_live, + actual_live: current_entity_live, + }); + } + if current_sequence != expected_sequence { + return Err(PersistenceError::ConcurrencyViolation { + expected: expected_sequence, + actual: current_sequence, + }); + } + + let mut accepted_rows = Vec::with_capacity(key_rows.len()); + for row in key_rows { + let slot = ( + tenant.to_string(), + entity_type.to_string(), + row.key_name.clone(), + row.key_hash.clone(), + ); + if let Some(existing) = inner.key_index.get(&slot) + && existing.0.as_str() != entity_id + { + return Err(PersistenceError::Storage(format!( + "duplicate declared key '{}' for {entity_type}: held by {}", + row.key_name, existing.0 + ))); + } + accepted_rows.push(slot); + } + + // Exact repair: purge every old row first, including when the current set is + // empty, then install the fully validated current claims. + inner.key_index.retain(|(t, et, _, _), (eid, _)| { + !(t.as_str() == tenant && et.as_str() == entity_type && eid.as_str() == entity_id) + }); + for slot in accepted_rows { + inner + .key_index + .insert(slot, (entity_id.to_string(), expected_sequence)); + } + Ok(()) +} + +pub(super) fn lookup( + inner: &SimEventStoreInner, + tenant: &str, + entity_type: &str, + key_name: &str, + key_hash: &str, +) -> Option { + let slot = ( + tenant.to_string(), + entity_type.to_string(), + key_name.to_string(), + key_hash.to_string(), + ); + inner + .key_index + .get(&slot) + .map(|(entity_id, sequence_nr)| EntityKeyLookup { + entity_id: entity_id.clone(), + sequence_nr: *sequence_nr, + }) +} + +pub(super) fn mark_backfilled( + inner: &mut SimEventStoreInner, + tenant: &str, + entity_type: &str, + key_set: &str, +) -> Result<(), PersistenceError> { + reconcile_key_contract_locked( + inner, + tenant, + entity_type, + Some(key_set), + KeyContractUse::Backfill, + )?; + inner.key_index_watermark.insert( + (tenant.to_string(), entity_type.to_string()), + key_set.to_string(), + ); + Ok(()) +} + +pub(super) fn backfilled_types(inner: &SimEventStoreInner, tenant: &str) -> Vec<(String, String)> { + inner + .key_index_watermark + .iter() + .filter(|((t, _), _)| t.as_str() == tenant) + .map(|((_, entity_type), key_set)| (entity_type.clone(), key_set.clone())) + .collect() +} + +pub(super) fn activated_contracts(inner: &SimEventStoreInner) -> Vec<(String, String)> { + inner.key_index_activated_contract.keys().cloned().collect() +} + +pub(super) fn reconciliation_revision( + inner: &SimEventStoreInner, + tenant: &str, + entity_type: &str, +) -> u64 { + inner + .key_index_contract + .get(&(tenant.to_string(), entity_type.to_string())) + .map(|(_, revision)| *revision) + .unwrap_or(0) +} + +pub(super) fn mark_backfilled_if_revision( + inner: &mut SimEventStoreInner, + tenant: &str, + entity_type: &str, + key_set: &str, + expected_revision: u64, +) -> bool { + let type_key = (tenant.to_string(), entity_type.to_string()); + let current = inner.key_index_contract.get(&type_key).cloned(); + if !matches!( + current, + Some((ref current_key_set, revision)) + if current_key_set == key_set && revision == expected_revision + ) { + return false; + } + inner + .key_index_watermark + .insert(type_key, key_set.to_string()); + true +} + +pub(super) fn keyed_entity_ids( + inner: &SimEventStoreInner, + tenant: &str, + entity_type: &str, +) -> Vec { + let mut ids = BTreeSet::new(); + for ((found_tenant, found_type, _, _), (entity_id, _)) in &inner.key_index { + if found_tenant == tenant && found_type == entity_type { + ids.insert(entity_id.clone()); + } + } + ids.into_iter().collect() +} + +pub(super) fn live_entity_ids( + inner: &SimEventStoreInner, + tenant: &str, + entity_type: &str, +) -> Vec { + let mut owners = BTreeSet::new(); + for persistence_id in inner.journals.keys().chain(inner.snapshots.keys()) { + if let Ok((found_tenant, found_type, entity_id)) = + parse_persistence_id_parts(persistence_id) + && found_tenant == tenant + && found_type == entity_type + && entity_is_live(inner, persistence_id) + { + owners.insert(entity_id.to_string()); + } + } + owners.into_iter().collect() +} + +fn entity_is_live(inner: &SimEventStoreInner, persistence_id: &str) -> bool { + let has_durable_state = + inner.journals.contains_key(persistence_id) || inner.snapshots.contains_key(persistence_id); + let deleted = inner.journals.get(persistence_id).is_some_and(|journal| { + journal + .iter() + .any(PersistenceEnvelope::transitions_to_deleted) + }); + has_durable_state && !deleted +} + +pub(super) fn reconciliation_entity_ids( + inner: &SimEventStoreInner, + tenant: &str, + entity_type: &str, +) -> Vec { + let mut owners = BTreeSet::new(); + for persistence_id in inner.journals.keys() { + if let Ok((found_tenant, found_type, entity_id)) = + parse_persistence_id_parts(persistence_id) + && found_tenant == tenant + && found_type == entity_type + { + owners.insert(entity_id.to_string()); + } + } + for persistence_id in inner.snapshots.keys() { + if let Ok((found_tenant, found_type, entity_id)) = + parse_persistence_id_parts(persistence_id) + && found_tenant == tenant + && found_type == entity_type + { + owners.insert(entity_id.to_string()); + } + } + for ((found_tenant, found_type, _, _), (entity_id, _)) in &inner.key_index { + if found_tenant == tenant && found_type == entity_type { + owners.insert(entity_id.clone()); + } + } + owners.into_iter().collect() +} diff --git a/crates/temper-store-sim/src/lib.rs b/crates/temper-store-sim/src/lib.rs index d52a96643..d28bda9b1 100644 --- a/crates/temper-store-sim/src/lib.rs +++ b/crates/temper-store-sim/src/lib.rs @@ -8,99 +8,31 @@ //! Server tests route this implementation through the `StorageStack` //! event-journal capability so production actor code runs unchanged. +mod append; +mod append_batch; +mod fault_injection; +mod key_index; +mod source_fence; + +use fault_injection::SimAppendPauseState; +pub use fault_injection::{DeterministicRng, SimAppendPause, SimFaultConfig}; + use std::collections::{BTreeMap, BTreeSet, VecDeque}; use std::sync::{Arc, Mutex}; -use std::time::Duration; +use tokio::sync::Notify; use temper_runtime::persistence::{ - EntityVectorCandidate, EntityVectorRow, EventStore, PersistenceAppend, PersistenceAppendResult, - PersistenceEnvelope, PersistenceError, + EntityKeyLookup, EntityVectorCandidate, EntityVectorRow, EventStore, IndexReconciliation, + JournalBoundary, KeyContractActivation, PersistenceAppend, PersistenceAppendResult, + PersistenceBatchIdempotency, PersistenceEnvelope, PersistenceError, SnapshotSourceFence, + is_state_materialization_event_for, }; use temper_runtime::tenant::parse_persistence_id_parts; -/// Fault injection configuration for simulation. -/// -/// Controls the probability of injected failures during event store operations. -/// All probabilities are in \[0.0, 1.0\]. -#[derive(Debug, Clone)] -pub struct SimFaultConfig { - /// Probability of a write failure on `append()`. - pub write_failure_prob: f64, - /// Probability of a spurious concurrency violation on `append()`. - pub concurrency_violation_prob: f64, - /// Probability of truncating journal on `read_events()`. - pub read_truncation_prob: f64, - /// Probability of snapshot save failure. - pub snapshot_failure_prob: f64, -} - -impl SimFaultConfig { - /// No fault injection — all operations succeed. - pub fn none() -> Self { - Self { - write_failure_prob: 0.0, - concurrency_violation_prob: 0.0, - read_truncation_prob: 0.0, - snapshot_failure_prob: 0.0, - } - } - - /// Heavy fault injection for stress testing. - pub fn heavy() -> Self { - Self { - write_failure_prob: 0.05, - concurrency_violation_prob: 0.02, - read_truncation_prob: 0.01, - snapshot_failure_prob: 0.03, - } - } -} - -impl Default for SimFaultConfig { - fn default() -> Self { - Self::none() - } -} - -/// Deterministic pseudo-random number generator for fault injection. -/// -/// Simple xorshift64 — fast, deterministic, good enough for fault injection. -/// Uses `BTreeMap` internally (DST compliance: deterministic iteration order). -#[derive(Debug, Clone)] -pub struct DeterministicRng { - state: u64, -} - -impl DeterministicRng { - /// Create a new RNG with the given seed. - pub fn new(seed: u64) -> Self { - Self { - state: if seed == 0 { 1 } else { seed }, - } - } - - /// Generate next u64. - pub fn next_u64(&mut self) -> u64 { - let mut x = self.state; - x ^= x << 13; - x ^= x >> 7; - x ^= x << 17; - self.state = x; - x - } - - /// Return true with the given probability \[0.0, 1.0\]. - pub fn chance(&mut self, prob: f64) -> bool { - if prob <= 0.0 { - return false; - } - if prob >= 1.0 { - return true; - } - let threshold = (prob * u64::MAX as f64) as u64; - self.next_u64() < threshold - } -} +use key_index::{ + KeyContractUse, activate_key_contract_locked, invalidate_coverage_for_snapshot_write_locked, + invalidate_coverage_for_unreconciled_append_locked, reconcile_key_contract_locked, +}; /// In-memory, deterministic event store for DST. /// @@ -119,6 +51,8 @@ pub struct SimEventStore { struct SimEventStoreInner { /// Event journals: persistence_id → Vec journals: BTreeMap>, + /// Content-bound claims co-committed with atomic append batches. + batch_idempotency: BTreeMap<(String, String), String>, /// Snapshots: persistence_id → (sequence_nr, snapshot_bytes) snapshots: BTreeMap)>, /// Immutable snapshot history: persistence_id → sequence_nr → snapshot bytes. @@ -142,23 +76,34 @@ struct SimEventStoreInner { /// (e.g. proving the key-index backfill treats an unreadable entity as /// `LoadFailed` and does not watermark its type). See `fail_next_reads`. pending_read_failures: BTreeMap, - /// One-shot append delays per `persistence_id`. - /// - /// Used by dispatch retry tests to deterministically model "the actor - /// persisted the transition, but the caller's ask timeout expired before - /// the reply arrived". - pending_append_delays: BTreeMap>, + /// Number of upcoming key-contract activation transactions that fail before + /// mutation. This deterministic fault proves persist-first publication stays + /// gated across the durable-spec/contract-activation crash boundary. + pending_key_activation_failures: usize, + /// One-shot deterministic post-commit append barriers per stream. + pending_postcommit_append_pauses: BTreeMap>, + /// One-shot deterministic pre-commit append barriers per stream. + pending_append_pauses: BTreeMap>, + /// One-shot deterministic pre-commit barriers for atomic append batches. + pending_batch_pauses: VecDeque, /// ADR-0153: declared key-index, co-committed with the journal under the same - /// lock. `(tenant, entity_type, key_name, key_hash) -> entity_id`. This is the + /// lock. `(tenant, entity_type, key_name, key_hash) -> (entity_id, sequence_nr)`. + /// This is the /// deterministic reference for the negative-existence access path the real /// stores maintain in `entity_key_index`. - key_index: BTreeMap<(String, String, String, String), String>, - /// ADR-0153 backfill watermark: `(tenant, entity_type) -> key_set` — each completed - /// type mapped to the sorted comma-joined declared key names the backfill covered. + key_index: BTreeMap<(String, String, String, String), (String, u64)>, + /// ADR-0153/0171 backfill watermark: `(tenant, entity_type) -> key_set` — each + /// completed type mapped to the versioned declared-key signature covered. /// The deterministic reference for the real stores' `key_index_backfill_watermark` /// table — gates authoritative keyed absence, and detects a key-set change so a /// newly-declared key re-keys instead of being treated as already complete. key_index_watermark: BTreeMap<(String, String), String>, + /// Monotonic key-contract state for ABA-safe watermark publication: + /// `(tenant, entity_type) -> (versioned signature, revision)`. + key_index_contract: BTreeMap<(String, String), (String, u64)>, + /// Latest spec-activated signature. Once present, delayed writers carrying + /// any other signature are rejected instead of redefining the contract. + key_index_activated_contract: BTreeMap<(String, String), (String, u64, String)>, /// ADR-0155: derived vector index, co-committed with the journal under the same /// lock. `(tenant, entity_type, decl_name, model_tag, entity_id) -> vector`. The /// deterministic reference for the real stores' `entity_vector_index` — the @@ -187,6 +132,7 @@ impl SimEventStore { Self { inner: Arc::new(Mutex::new(SimEventStoreInner { journals: BTreeMap::new(), + batch_idempotency: BTreeMap::new(), snapshots: BTreeMap::new(), snapshot_history: BTreeMap::new(), event_segments: BTreeMap::new(), @@ -194,9 +140,14 @@ impl SimEventStore { faults, pending_concurrency_violations: BTreeMap::new(), pending_read_failures: BTreeMap::new(), - pending_append_delays: BTreeMap::new(), + pending_key_activation_failures: 0, + pending_postcommit_append_pauses: BTreeMap::new(), + pending_append_pauses: BTreeMap::new(), + pending_batch_pauses: VecDeque::new(), key_index: BTreeMap::new(), key_index_watermark: BTreeMap::new(), + key_index_contract: BTreeMap::new(), + key_index_activated_contract: BTreeMap::new(), vector_index: BTreeMap::new(), vector_index_watermark: BTreeMap::new(), })), @@ -239,6 +190,16 @@ impl SimEventStore { } } + /// Fail the next `count` key-contract activation transactions before any + /// contract, watermark, or ownership row is mutated. `count == 0` clears + /// the deterministic fault. + pub fn fail_next_key_activations(&self, count: usize) { + self.inner + .lock() + .expect("SimEventStore lock poisoned") // ci-ok: infallible lock + .pending_key_activation_failures = count; + } + /// Return the current count of pending injected concurrency violations for /// `persistence_id`. Zero if none are queued. pub fn pending_concurrency_violations(&self, persistence_id: &str) -> u64 { @@ -250,20 +211,55 @@ impl SimEventStore { .unwrap_or(0) } - /// Delay the next append for `persistence_id` by `delay`. - /// - /// The delay is consumed once. Multiple calls queue multiple delays in - /// FIFO order. - pub fn inject_append_delay(&self, persistence_id: &str, delay: Duration) { - let mut inner = self - .inner + /// Pause the next append for `persistence_id` after its journal/index + /// mutation commits but before the append future acknowledges success. + /// Tests control the interleaving explicitly through the returned barrier; + /// no wall-clock timer participates in simulation. + pub fn inject_postcommit_append_pause(&self, persistence_id: &str) -> SimAppendPause { + let state = SimAppendPauseState { + reached: Arc::new(Notify::new()), + resume: Arc::new(Notify::new()), + }; + self.inner .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); - inner - .pending_append_delays + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .pending_postcommit_append_pauses .entry(persistence_id.to_string()) .or_default() - .push_back(delay); + .push_back(state.clone()); + SimAppendPause { state } + } + + /// Pause the next append for `persistence_id` immediately before it takes + /// the store lock or mutates journal/index state. + pub fn inject_precommit_append_pause(&self, persistence_id: &str) -> SimAppendPause { + let state = SimAppendPauseState { + reached: Arc::new(Notify::new()), + resume: Arc::new(Notify::new()), + }; + self.inner + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .pending_append_pauses + .entry(persistence_id.to_string()) + .or_default() + .push_back(state.clone()); + SimAppendPause { state } + } + + /// Pause the next atomic append batch before it takes the store lock or + /// mutates any journal/index state. + pub fn inject_precommit_batch_pause(&self) -> SimAppendPause { + let state = SimAppendPauseState { + reached: Arc::new(Notify::new()), + resume: Arc::new(Notify::new()), + }; + self.inner + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .pending_batch_pauses + .push_back(state.clone()); + SimAppendPause { state } } /// Create a SimEventStore with no fault injection. @@ -348,14 +344,43 @@ impl std::fmt::Debug for SimEventStore { } impl EventStore for SimEventStore { + fn supports_authoritative_key_index(&self) -> bool { + true + } + + async fn batch_idempotency_committed( + &self, + claim: &PersistenceBatchIdempotency, + ) -> Result { + let inner = self.inner.lock().expect("SimEventStore lock poisoned"); + let claim_key = (claim.persistence_id.clone(), claim.idempotency_key.clone()); + let Some(committed_hash) = inner.batch_idempotency.get(&claim_key) else { + return Ok(false); + }; + if committed_hash != &claim.intent_hash { + return Err(PersistenceError::Storage(format!( + "atomic batch idempotency key '{}' was reused with a different intent", + claim.idempotency_key + ))); + } + Ok(true) + } + async fn append( &self, persistence_id: &str, expected_sequence: u64, events: &[PersistenceEnvelope], ) -> Result { - self.append_with_index_rows(persistence_id, expected_sequence, events, &[], &[], false) - .await + self.append_with_index_rows( + persistence_id, + expected_sequence, + events, + &[], + &[], + IndexReconciliation::default(), + ) + .await } async fn append_with_index_rows( @@ -365,224 +390,17 @@ impl EventStore for SimEventStore { events: &[PersistenceEnvelope], key_rows: &[temper_runtime::persistence::EntityKeyRow], vector_rows: &[EntityVectorRow], - reconcile_vectors: bool, + reconciliation: IndexReconciliation, ) -> Result { - let append_delay = { - let mut inner = self - .inner - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); - let delay = inner - .pending_append_delays - .get_mut(persistence_id) - .and_then(VecDeque::pop_front); - if inner - .pending_append_delays - .get(persistence_id) - .is_some_and(VecDeque::is_empty) - { - inner.pending_append_delays.remove(persistence_id); - } - delay - }; - if let Some(delay) = append_delay - && !delay.is_zero() - { - tokio::time::sleep(delay).await; - } - - let mut inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock - - // Deterministic one-shot injection (see `inject_concurrency_violations`). - // Consumes one counter per call; falls back to normal flow once drained. - // - // The reported `actual` equals `expected_sequence` — the journal has - // not actually moved, so an authoritative replay will land back at - // `expected_sequence`. Any code that asserts - // `post_replay_sequence >= actual` still holds without this injection - // lying about journal state. - let pending_cv = inner - .pending_concurrency_violations - .get(persistence_id) - .copied() - .unwrap_or(0); - if pending_cv > 0 { - if pending_cv == 1 { - inner.pending_concurrency_violations.remove(persistence_id); - } else { - inner - .pending_concurrency_violations - .insert(persistence_id.to_string(), pending_cv - 1); - } - return Err(PersistenceError::ConcurrencyViolation { - expected: expected_sequence, - actual: expected_sequence, - }); - } - - // Fault injection: spurious concurrency violation (probabilistic). - let cv_prob = inner.faults.concurrency_violation_prob; - if inner.rng.chance(cv_prob) { - return Err(PersistenceError::ConcurrencyViolation { - expected: expected_sequence, - actual: expected_sequence.wrapping_add(1), - }); - } - - // Fault injection: write failure. - let wf_prob = inner.faults.write_failure_prob; - if inner.rng.chance(wf_prob) { - return Err(PersistenceError::Storage( - "SimEventStore: injected write failure".into(), - )); - } - - // Check optimistic concurrency. - let current_seq = inner - .journals - .get(persistence_id) - .and_then(|journal| journal.last().map(|e| e.sequence_nr)) - .unwrap_or(0); - if current_seq != expected_sequence { - return Err(PersistenceError::ConcurrencyViolation { - expected: expected_sequence, - actual: current_seq, - }); - } - - // ADR-0153: validate declared-key uniqueness BEFORE writing the journal, so - // a reject is atomic — the journal must not advance on a rejected co-commit. - // A *different* entity already holding the key is the violation. - if !key_rows.is_empty() { - let mut parts = persistence_id.splitn(3, ':'); - let tenant = parts.next().unwrap_or(""); - let entity_type = parts.next().unwrap_or(""); - let entity_id = parts.next().unwrap_or(""); - for row in key_rows { - if let Some(existing) = inner.key_index.get(&( - tenant.to_string(), - entity_type.to_string(), - row.key_name.clone(), - row.key_hash.clone(), - )) && existing.as_str() != entity_id - { - return Err(PersistenceError::Storage(format!( - "duplicate declared key '{}' for {entity_type}: held by {existing}", - row.key_name - ))); - } - } - } - - let mut new_seq = expected_sequence; - let mut stored_events = Vec::with_capacity(events.len()); - for event in events { - new_seq += 1; - // Store with correct sequence number (ignore the one in the envelope, - // use monotonic counter like the real stores do). - let mut stored = event.clone(); - stored.sequence_nr = new_seq; - stored_events.push(stored); - } - inner - .journals - .entry(persistence_id.to_string()) - .or_default() - .extend(stored_events); - - let segments = inner - .event_segments - .entry(persistence_id.to_string()) - .or_insert_with(|| { - vec![SimEventSegment { - segment_index: 0, - start_sequence_nr: (expected_sequence + 1).max(1), - end_sequence_nr: None, - snapshot_sequence: None, - event_count: 0, - sealed: false, - }] - }); - if segments.last().map(|s| s.sealed).unwrap_or(true) { - let next_index = segments.last().map(|s| s.segment_index + 1).unwrap_or(0); - segments.push(SimEventSegment { - segment_index: next_index, - start_sequence_nr: (expected_sequence + 1).max(1), - end_sequence_nr: None, - snapshot_sequence: None, - event_count: 0, - sealed: false, - }); - } - if new_seq > expected_sequence { - let active_segment = segments - .last_mut() - .expect("segments must contain an active segment"); - active_segment.end_sequence_nr = Some(new_seq); - active_segment.event_count = new_seq - .saturating_sub(active_segment.start_sequence_nr) - .saturating_add(1); - } - - // ADR-0153: co-commit the declared key-index rows under the SAME lock as - // the journal write above (uniqueness was validated before the journal, so - // this only mutates — never fails). A keyed read is therefore consistent - // with the journal: the negative-existence access path. - if !key_rows.is_empty() { - let mut parts = persistence_id.splitn(3, ':'); - let tenant = parts.next().unwrap_or(""); - let entity_type = parts.next().unwrap_or(""); - let entity_id = parts.next().unwrap_or(""); - for row in key_rows { - // Drop the entity's prior row for this key_name (the value may have - // changed), then claim the new (key_name, key_hash) -> entity_id. - inner.key_index.retain(|(t, et, kn, _), eid| { - !(t.as_str() == tenant - && et.as_str() == entity_type - && kn.as_str() == row.key_name.as_str() - && eid.as_str() == entity_id) - }); - inner.key_index.insert( - ( - tenant.to_string(), - entity_type.to_string(), - row.key_name.clone(), - row.key_hash.clone(), - ), - entity_id.to_string(), - ); - } - } - - // ADR-0155: co-commit the derived vector-index rows under the SAME lock as - // the journal write. When the entity's type declares vector paths - // (`reconcile_vectors`), DELETE all of the entity's rows first, then insert - // the current ones — so a delete transition or a cleared vector/model - // property (empty `vector_rows`) purges the stale rows instead of leaving - // them to rank forever. No uniqueness constraint — vectors are derived state. - if reconcile_vectors { - let mut parts = persistence_id.splitn(3, ':'); - let tenant = parts.next().unwrap_or(""); - let entity_type = parts.next().unwrap_or(""); - let entity_id = parts.next().unwrap_or(""); - inner.vector_index.retain(|(t, et, _, _, eid), _| { - !(t.as_str() == tenant && et.as_str() == entity_type && eid.as_str() == entity_id) - }); - for row in vector_rows { - inner.vector_index.insert( - ( - tenant.to_string(), - entity_type.to_string(), - row.decl_name.clone(), - row.model_tag.clone(), - entity_id.to_string(), - ), - row.vector.clone(), - ); - } - } - - Ok(new_seq) + self.append_with_index_rows_inner( + persistence_id, + expected_sequence, + events, + key_rows, + vector_rows, + reconciliation, + ) + .await } async fn backfill_entity_keys( @@ -590,32 +408,20 @@ impl EventStore for SimEventStore { tenant: &str, entity_type: &str, entity_id: &str, + expected_sequence: u64, + contract_fence: temper_runtime::persistence::KeyIndexBackfillFence<'_>, key_rows: &[temper_runtime::persistence::EntityKeyRow], ) -> Result<(), PersistenceError> { let mut inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock - for row in key_rows { - let slot = ( - tenant.to_string(), - entity_type.to_string(), - row.key_name.clone(), - row.key_hash.clone(), - ); - match inner.key_index.get(&slot) { - // A different entity holds it — pre-existing conflict; skip (don't - // clobber, don't fail the backfill). - Some(existing) if existing.as_str() != entity_id => continue, - _ => { - inner.key_index.retain(|(t, et, kn, _), eid| { - !(t.as_str() == tenant - && et.as_str() == entity_type - && kn.as_str() == row.key_name.as_str() - && eid.as_str() == entity_id) - }); - inner.key_index.insert(slot, entity_id.to_string()); - } - } - } - Ok(()) + key_index::backfill_entity_keys( + &mut inner, + tenant, + entity_type, + entity_id, + expected_sequence, + contract_fence, + key_rows, + ) } async fn lookup_by_key( @@ -626,13 +432,27 @@ impl EventStore for SimEventStore { key_hash: &str, ) -> Result, PersistenceError> { let inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock - let slot = ( - tenant.to_string(), - entity_type.to_string(), - key_name.to_string(), - key_hash.to_string(), - ); - Ok(inner.key_index.get(&slot).cloned()) + Ok( + key_index::lookup(&inner, tenant, entity_type, key_name, key_hash) + .map(|lookup| lookup.entity_id), + ) + } + + async fn lookup_by_key_with_sequence( + &self, + tenant: &str, + entity_type: &str, + key_name: &str, + key_hash: &str, + ) -> Result, PersistenceError> { + let inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock + Ok(key_index::lookup( + &inner, + tenant, + entity_type, + key_name, + key_hash, + )) } async fn mark_key_index_backfilled( @@ -642,13 +462,7 @@ impl EventStore for SimEventStore { key_set: &str, ) -> Result<(), PersistenceError> { let mut inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock - // Overwrite the covered key-set (a re-key after a key-set change replaces the - // stale set), mirroring the Postgres upsert. - inner.key_index_watermark.insert( - (tenant.to_string(), entity_type.to_string()), - key_set.to_string(), - ); - Ok(()) + key_index::mark_backfilled(&mut inner, tenant, entity_type, key_set) } async fn key_index_backfilled_types( @@ -656,27 +470,143 @@ impl EventStore for SimEventStore { tenant: &str, ) -> Result, PersistenceError> { let inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock - Ok(inner - .key_index_watermark - .iter() - .filter(|((t, _), _)| t.as_str() == tenant) - .map(|((_, et), key_set)| (et.clone(), key_set.clone())) - .collect()) + Ok(key_index::backfilled_types(&inner, tenant)) } - async fn keyed_entity_ids_for_type( + async fn key_index_activated_contracts( + &self, + ) -> Result, PersistenceError> { + let inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock + Ok(key_index::activated_contracts(&inner)) + } + + async fn key_index_reconciliation_revision( &self, tenant: &str, entity_type: &str, - ) -> Result, PersistenceError> { + ) -> Result { let inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock - let mut ids: BTreeSet = BTreeSet::new(); - for ((t, et, _, _), entity_id) in inner.key_index.iter() { - if t.as_str() == tenant && et.as_str() == entity_type { - ids.insert(entity_id.clone()); + Ok(key_index::reconciliation_revision( + &inner, + tenant, + entity_type, + )) + } + + async fn begin_key_index_backfill( + &self, + tenant: &str, + entity_type: &str, + key_set: &str, + ) -> Result { + let mut inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock + reconcile_key_contract_locked( + &mut inner, + tenant, + entity_type, + Some(key_set), + KeyContractUse::Backfill, + ) + } + + async fn activate_key_index_contract( + &self, + tenant: &str, + entity_type: &str, + key_set: &str, + purge_existing_rows: bool, + ) -> Result { + let mut inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock + if inner.pending_key_activation_failures > 0 { + inner.pending_key_activation_failures -= 1; + return Err(PersistenceError::Storage( + "SimEventStore: injected key activation failure".to_string(), + )); + } + activate_key_contract_locked( + &mut inner, + tenant, + entity_type, + key_set, + key_set, + purge_existing_rows, + ) + } + + async fn activate_key_index_contracts( + &self, + tenant: &str, + activations: &[KeyContractActivation], + ) -> Result, PersistenceError> { + let mut inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock + if inner.pending_key_activation_failures > 0 { + inner.pending_key_activation_failures -= 1; + return Err(PersistenceError::Storage( + "SimEventStore: injected key activation failure".to_string(), + )); + } + let mut seen = BTreeSet::new(); + for activation in activations { + if !seen.insert(activation.entity_type.as_str()) { + return Err(PersistenceError::Storage(format!( + "SimEventStore: duplicate key activation for {tenant}:{}", + activation.entity_type + ))); } } - Ok(ids.into_iter().collect()) + let contract_before = inner.key_index_contract.clone(); + let activated_before = inner.key_index_activated_contract.clone(); + let watermark_before = inner.key_index_watermark.clone(); + let rows_before = inner.key_index.clone(); + let mut epochs = BTreeMap::new(); + for activation in activations { + match activate_key_contract_locked( + &mut inner, + tenant, + &activation.entity_type, + &activation.key_set, + &activation.spec_fingerprint, + activation.purge_existing_rows, + ) { + Ok(epoch) => { + epochs.insert(activation.entity_type.clone(), epoch); + } + Err(error) => { + inner.key_index_contract = contract_before; + inner.key_index_activated_contract = activated_before; + inner.key_index_watermark = watermark_before; + inner.key_index = rows_before; + return Err(error); + } + } + } + Ok(epochs) + } + + async fn mark_key_index_backfilled_if_revision( + &self, + tenant: &str, + entity_type: &str, + key_set: &str, + expected_revision: u64, + ) -> Result { + let mut inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock + Ok(key_index::mark_backfilled_if_revision( + &mut inner, + tenant, + entity_type, + key_set, + expected_revision, + )) + } + + async fn keyed_entity_ids_for_type( + &self, + tenant: &str, + entity_type: &str, + ) -> Result, PersistenceError> { + let inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock + Ok(key_index::keyed_entity_ids(&inner, tenant, entity_type)) } async fn backfill_entity_vectors( @@ -784,97 +714,7 @@ impl EventStore for SimEventStore { &self, appends: &[PersistenceAppend], ) -> Result, PersistenceError> { - if appends.is_empty() { - return Ok(Vec::new()); - } - - let mut inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock - - let mut seen = std::collections::BTreeSet::new(); - for append in appends { - if !seen.insert(append.persistence_id.as_str()) { - return Err(PersistenceError::Storage(format!( - "SimEventStore: duplicate persistence_id '{}' in append_batch", - append.persistence_id - ))); - } - } - - for append in appends { - let pending_cv = inner - .pending_concurrency_violations - .get(&append.persistence_id) - .copied() - .unwrap_or(0); - if pending_cv > 0 { - if pending_cv == 1 { - inner - .pending_concurrency_violations - .remove(&append.persistence_id); - } else { - inner - .pending_concurrency_violations - .insert(append.persistence_id.clone(), pending_cv - 1); - } - return Err(PersistenceError::ConcurrencyViolation { - expected: append.expected_sequence, - actual: append.expected_sequence, - }); - } - } - - // Fault injection happens before mutation so a batch either writes - // every stream or no stream. - let cv_prob = inner.faults.concurrency_violation_prob; - if inner.rng.chance(cv_prob) { - let first = &appends[0]; - return Err(PersistenceError::ConcurrencyViolation { - expected: first.expected_sequence, - actual: first.expected_sequence.wrapping_add(1), - }); - } - let wf_prob = inner.faults.write_failure_prob; - if inner.rng.chance(wf_prob) { - return Err(PersistenceError::Storage( - "SimEventStore: injected batch write failure".into(), - )); - } - - for append in appends { - let current_seq = inner - .journals - .get(&append.persistence_id) - .and_then(|journal| journal.last()) - .map(|event| event.sequence_nr) - .unwrap_or(0); - if current_seq != append.expected_sequence { - return Err(PersistenceError::ConcurrencyViolation { - expected: append.expected_sequence, - actual: current_seq, - }); - } - } - - let mut results = Vec::with_capacity(appends.len()); - for append in appends { - let journal = inner - .journals - .entry(append.persistence_id.clone()) - .or_default(); - let mut new_seq = append.expected_sequence; - for event in &append.events { - new_seq += 1; - let mut stored = event.clone(); - stored.sequence_nr = new_seq; - journal.push(stored); - } - results.push(PersistenceAppendResult { - persistence_id: append.persistence_id.clone(), - sequence_nr: new_seq, - }); - } - - Ok(results) + self.append_batch_inner(appends).await } async fn read_events( @@ -917,73 +757,93 @@ impl EventStore for SimEventStore { Ok(events) } - async fn save_snapshot( + async fn read_events_page( &self, persistence_id: &str, - sequence_nr: u64, - snapshot: &[u8], - ) -> Result<(), PersistenceError> { + from_sequence: u64, + through_sequence: u64, + limit: usize, + ) -> Result, PersistenceError> { + assert!(limit > 0, "event page limit must be positive"); + assert!( + through_sequence >= from_sequence, + "event page boundary must not precede its cursor" + ); let mut inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock - // Fault injection: snapshot save failure. - let sf_prob = inner.faults.snapshot_failure_prob; - if inner.rng.chance(sf_prob) { - return Err(PersistenceError::Storage( - "SimEventStore: injected snapshot failure".into(), - )); + if let Some(remaining) = inner.pending_read_failures.get_mut(persistence_id) { + *remaining -= 1; + let cleared = *remaining == 0; + if cleared { + inner.pending_read_failures.remove(persistence_id); + } + return Err(PersistenceError::Storage(format!( + "injected read failure for {persistence_id}" + ))); } - inner - .snapshots - .insert(persistence_id.to_string(), (sequence_nr, snapshot.to_vec())); - inner - .snapshot_history - .entry(persistence_id.to_string()) - .or_default() - .insert(sequence_nr, snapshot.to_vec()); - let segments = inner - .event_segments - .entry(persistence_id.to_string()) - .or_insert_with(|| { - vec![SimEventSegment { - segment_index: 0, - start_sequence_nr: 1, - end_sequence_nr: Some(sequence_nr), - snapshot_sequence: None, - event_count: sequence_nr, - sealed: false, - }] - }); - if segments.last().map(|s| s.sealed).unwrap_or(true) { - let idx = segments.last().map(|s| s.segment_index + 1).unwrap_or(0); - segments.push(SimEventSegment { - segment_index: idx, - start_sequence_nr: 1, - end_sequence_nr: Some(sequence_nr), - snapshot_sequence: None, - event_count: sequence_nr, - sealed: false, - }); + let journal = match inner.journals.get(persistence_id) { + Some(journal) => journal, + None => return Ok(Vec::new()), + }; + let mut events = journal + .iter() + .filter(|event| { + event.sequence_nr > from_sequence && event.sequence_nr <= through_sequence + }) + .take(limit) + .cloned() + .collect::>(); + + let read_truncation_probability = inner.faults.read_truncation_prob; + if !events.is_empty() && inner.rng.chance(read_truncation_probability) { + let truncate_at = (inner.rng.next_u64() as usize) % events.len(); + events.truncate(truncate_at.max(1)); } - let active = segments - .last_mut() - .expect("segments must contain an active segment"); - active.end_sequence_nr = Some(sequence_nr); - active.snapshot_sequence = Some(sequence_nr); - active.event_count = sequence_nr - .saturating_sub(active.start_sequence_nr) - .saturating_add(1); - active.sealed = true; - let next_index = active.segment_index + 1; - segments.push(SimEventSegment { - segment_index: next_index, - start_sequence_nr: sequence_nr + 1, - end_sequence_nr: None, - snapshot_sequence: None, - event_count: 0, - sealed: false, - }); - Ok(()) + Ok(events) + } + + async fn journal_boundary( + &self, + persistence_id: &str, + ) -> Result { + let inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock + let Some(journal) = inner.journals.get(persistence_id) else { + return Ok(JournalBoundary::default()); + }; + Ok(JournalBoundary { + latest_sequence: journal.last().map(|event| event.sequence_nr).unwrap_or(0), + first_terminal_sequence: journal + .iter() + .find(|event| event.transitions_to_deleted()) + .map(|event| event.sequence_nr), + }) + } + + async fn save_snapshot( + &self, + persistence_id: &str, + sequence_nr: u64, + snapshot: &[u8], + ) -> Result<(), PersistenceError> { + self.save_snapshot_locked( + persistence_id, + sequence_nr, + snapshot, + &SnapshotSourceFence::Unchecked, + None, + ) + } + + async fn save_snapshot_if_source( + &self, + persistence_id: &str, + sequence_nr: u64, + snapshot: &[u8], + source: &SnapshotSourceFence, + key_contract: Option<&str>, + ) -> Result<(), PersistenceError> { + self.save_snapshot_locked(persistence_id, sequence_nr, snapshot, source, key_contract) } async fn load_snapshot( @@ -1022,20 +882,64 @@ impl EventStore for SimEventStore { entity_type: &str, ) -> Result, PersistenceError> { let inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock - let mut result = Vec::new(); - let mut seen = std::collections::BTreeSet::new(); + Ok(key_index::live_entity_ids(&inner, tenant, entity_type)) + } - for persistence_id in inner.journals.keys() { - if let Ok((t, found_type, entity_id)) = parse_persistence_id_parts(persistence_id) - && t == tenant - && found_type == entity_type - && seen.insert(entity_id.to_string()) - { - result.push(entity_id.to_string()); - } - } + async fn list_entity_ids_for_key_reconciliation( + &self, + tenant: &str, + entity_type: &str, + ) -> Result, PersistenceError> { + let inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock + Ok(key_index::reconciliation_entity_ids( + &inner, + tenant, + entity_type, + )) + } - Ok(result) + async fn list_key_reconciliation_page( + &self, + tenant: &str, + entity_type: &str, + after_entity_id: Option<&str>, + through_entity_id: &str, + limit: usize, + ) -> Result, PersistenceError> { + assert!(limit > 0, "key reconciliation page limit must be positive"); + let inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock + let live = key_index::live_entity_ids(&inner, tenant, entity_type) + .into_iter() + .collect::>(); + Ok( + key_index::reconciliation_entity_ids(&inner, tenant, entity_type) + .into_iter() + .filter(|entity_id| { + after_entity_id.is_none_or(|cursor| entity_id.as_str() > cursor) + && entity_id.as_str() <= through_entity_id + }) + .take(limit) + .map( + |entity_id| temper_runtime::persistence::KeyReconciliationEntity { + is_live: live.contains(&entity_id), + entity_id, + }, + ) + .collect(), + ) + } + + async fn key_reconciliation_boundary( + &self, + tenant: &str, + entity_type: &str, + ) -> Result, PersistenceError> { + let inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock + Ok( + key_index::reconciliation_entity_ids(&inner, tenant, entity_type) + .into_iter() + .max(), + ) } } diff --git a/crates/temper-store-sim/src/source_fence.rs b/crates/temper-store-sim/src/source_fence.rs new file mode 100644 index 000000000..cfc60971c --- /dev/null +++ b/crates/temper-store-sim/src/source_fence.rs @@ -0,0 +1,224 @@ +//! Snapshot-source fences and segment materialization. + +use super::*; + +pub(super) fn snapshot_source_matches( + current: Option<&(u64, Vec)>, + expected: &SnapshotSourceFence, +) -> bool { + match expected { + SnapshotSourceFence::Unchecked => true, + SnapshotSourceFence::Absent => current.is_none(), + SnapshotSourceFence::Exact { sequence_nr, state } => current.is_some_and(|current| { + current.0 == *sequence_nr && current.1.as_slice() == state.as_slice() + }), + } +} + +pub(super) fn append_retires_snapshot( + expected_sequence: u64, + events: &[PersistenceEnvelope], + snapshot_source: &SnapshotSourceFence, + entity_type: &str, + entity_id: &str, +) -> bool { + expected_sequence == 0 + && matches!(snapshot_source, SnapshotSourceFence::Exact { .. }) + && events + .first() + .is_some_and(|event| is_state_materialization_event_for(event, entity_type, entity_id)) +} + +pub(super) fn update_segments_after_append_locked( + inner: &mut SimEventStoreInner, + persistence_id: &str, + expected_sequence: u64, + new_sequence: u64, +) { + if new_sequence == expected_sequence { + return; + } + let segments = inner + .event_segments + .entry(persistence_id.to_string()) + .or_default(); + let invalid_open = segments.last().is_some_and(|segment| { + !segment.sealed && segment.start_sequence_nr > expected_sequence.saturating_add(1) + }); + if expected_sequence == 0 || invalid_open { + // Snapshot-only generations have no journal segment. Reset legacy + // metadata that extends beyond the journal before assigning events. + segments.clear(); + } + if segments.is_empty() && expected_sequence > 0 { + segments.push(SimEventSegment { + segment_index: 0, + start_sequence_nr: 1, + end_sequence_nr: Some(expected_sequence), + snapshot_sequence: None, + event_count: expected_sequence, + sealed: false, + }); + } + if segments + .last() + .map(|segment| segment.sealed) + .unwrap_or(true) + { + let next_index = segments + .last() + .map(|segment| segment.segment_index + 1) + .unwrap_or(0); + segments.push(SimEventSegment { + segment_index: next_index, + start_sequence_nr: expected_sequence.saturating_add(1).max(1), + end_sequence_nr: None, + snapshot_sequence: None, + event_count: 0, + sealed: false, + }); + } + let active = segments + .last_mut() + .expect("a non-empty append must have an active segment"); + active.end_sequence_nr = Some(new_sequence); + active.event_count = new_sequence + .saturating_sub(active.start_sequence_nr) + .saturating_add(1); +} + +impl SimEventStore { + pub(super) fn save_snapshot_locked( + &self, + persistence_id: &str, + sequence_nr: u64, + snapshot: &[u8], + source: &SnapshotSourceFence, + key_contract: Option<&str>, + ) -> Result<(), PersistenceError> { + let (tenant, entity_type, entity_id) = + parse_persistence_id_parts(persistence_id).map_err(PersistenceError::Storage)?; + let mut inner = self.inner.lock().expect("SimEventStore lock poisoned"); // ci-ok: infallible lock + + if !snapshot_source_matches(inner.snapshots.get(persistence_id), source) { + return Err(PersistenceError::SnapshotGenerationChanged); + } + let journal = inner.journals.get(persistence_id); + let journal_high_water = journal + .and_then(|events| events.last()) + .map(|event| event.sequence_nr) + .unwrap_or(0); + if matches!(source, SnapshotSourceFence::Unchecked) + && journal_high_water > 0 + && journal + .and_then(|events| events.first()) + .is_some_and(|event| { + is_state_materialization_event_for(event, entity_type, entity_id) + }) + { + // The first journal append already consumed the migration snapshot. + // A delayed unchecked writer belongs to the retired snapshot-only + // generation regardless of whether the journal has since caught up + // with its numeric sequence. + return Ok(()); + } + let current_snapshot = inner.snapshots.get(persistence_id); + let snapshot_is_noop = current_snapshot.is_some_and(|(stored_sequence, stored)| { + *stored_sequence > sequence_nr + || (*stored_sequence == sequence_nr && stored.as_slice() == snapshot) + }); + if snapshot_is_noop { + return Ok(()); + } + let same_sequence_replacement = + current_snapshot.is_some_and(|(stored_sequence, _)| *stored_sequence == sequence_nr); + + // Fault injection: snapshot save failure. + let sf_prob = inner.faults.snapshot_failure_prob; + if inner.rng.chance(sf_prob) { + return Err(PersistenceError::Storage( + "SimEventStore: injected snapshot failure".into(), + )); + } + + if let Some(key_contract) = key_contract { + // Source mismatches and injected failures return before touching + // the contract. From this point onward all updates are infallible + // map mutations, mirroring one transactional store commit. + reconcile_key_contract_locked( + &mut inner, + tenant, + entity_type, + Some(key_contract), + KeyContractUse::LiveWrite, + )?; + } else { + invalidate_coverage_for_snapshot_write_locked(&mut inner, persistence_id)?; + } + + inner + .snapshots + .insert(persistence_id.to_string(), (sequence_nr, snapshot.to_vec())); + inner + .snapshot_history + .entry(persistence_id.to_string()) + .or_default() + .insert(sequence_nr, snapshot.to_vec()); + if same_sequence_replacement { + return Ok(()); + } + if journal_high_water == 0 { + inner.event_segments.remove(persistence_id); + return Ok(()); + } + if sequence_nr > journal_high_water { + // Snapshots accelerate recovery but do not invent journal history. + // A migration baseline ahead of the journal is not an event boundary. + return Ok(()); + } + let segments = inner + .event_segments + .entry(persistence_id.to_string()) + .or_insert_with(|| { + vec![SimEventSegment { + segment_index: 0, + start_sequence_nr: 1, + end_sequence_nr: (journal_high_water > 0).then_some(journal_high_water), + snapshot_sequence: None, + event_count: journal_high_water, + sealed: false, + }] + }); + if segments.last().map(|s| s.sealed).unwrap_or(true) { + let idx = segments.last().map(|s| s.segment_index + 1).unwrap_or(0); + segments.push(SimEventSegment { + segment_index: idx, + start_sequence_nr: 1, + end_sequence_nr: (journal_high_water > 0).then_some(journal_high_water), + snapshot_sequence: None, + event_count: journal_high_water, + sealed: false, + }); + } + let active = segments + .last_mut() + .expect("segments must contain an active segment"); + active.end_sequence_nr = Some(sequence_nr); + active.snapshot_sequence = Some(sequence_nr); + active.event_count = sequence_nr + .saturating_sub(active.start_sequence_nr) + .saturating_add(1); + active.sealed = true; + let next_index = active.segment_index + 1; + let tail_end = (journal_high_water > sequence_nr).then_some(journal_high_water); + segments.push(SimEventSegment { + segment_index: next_index, + start_sequence_nr: sequence_nr + 1, + end_sequence_nr: tail_end, + snapshot_sequence: None, + event_count: journal_high_water.saturating_sub(sequence_nr), + sealed: false, + }); + Ok(()) + } +} diff --git a/crates/temper-store-sim/src/tests.rs b/crates/temper-store-sim/src/tests.rs index 9e076ff0e..22b470597 100644 --- a/crates/temper-store-sim/src/tests.rs +++ b/crates/temper-store-sim/src/tests.rs @@ -1,5 +1,9 @@ use super::*; -use temper_runtime::persistence::EventMetadata; +use temper_runtime::persistence::{ + EventMetadata, STATE_MATERIALIZATION_EVENT_TYPE, STATE_MATERIALIZATION_SCHEMA, +}; + +mod key_index; fn test_envelope(seq: u64, event_type: &str) -> PersistenceEnvelope { PersistenceEnvelope { @@ -16,6 +20,30 @@ fn test_envelope(seq: u64, event_type: &str) -> PersistenceEnvelope { } } +fn materialization_envelope(entity_type: &str, entity_id: &str) -> PersistenceEnvelope { + let mut envelope = test_envelope(0, STATE_MATERIALIZATION_EVENT_TYPE); + envelope.payload = serde_json::json!({ + "schema": STATE_MATERIALIZATION_SCHEMA, + "state": { + "entity_type": entity_type, + "entity_id": entity_id, + "status": "Ready", + "item_count": 0, + "counters": {}, + "booleans": {}, + "lists": {}, + "fields": {"Id": entity_id, "Status": "Ready"}, + "events": [], + "total_event_count": 0, + "events_since_snapshot": 0, + "last_snapshot_sequence_nr": 0, + "sequence_nr": 0, + "processed_idempotency_keys": {}, + } + }); + envelope +} + #[tokio::test] async fn append_and_read_roundtrip() { let store = SimEventStore::no_faults(42); @@ -62,11 +90,21 @@ async fn append_batch_commits_multiple_journals_atomically() { persistence_id: "default:Order:ord-a".to_string(), expected_sequence: 0, events: vec![test_envelope(0, "Created")], + key_rows: Vec::new(), + reconcile_keys: false, + key_set_signature: None, + snapshot_source: Default::default(), + batch_idempotency: None, }, PersistenceAppend { persistence_id: "default:Order:ord-b".to_string(), expected_sequence: 0, events: vec![test_envelope(0, "Created"), test_envelope(0, "Submitted")], + key_rows: Vec::new(), + reconcile_keys: false, + key_set_signature: None, + snapshot_source: Default::default(), + batch_idempotency: None, }, ]; @@ -78,10 +116,12 @@ async fn append_batch_commits_multiple_journals_atomically() { PersistenceAppendResult { persistence_id: "default:Order:ord-a".to_string(), sequence_nr: 1, + batch_already_applied: false, }, PersistenceAppendResult { persistence_id: "default:Order:ord-b".to_string(), sequence_nr: 2, + batch_already_applied: false, }, ] ); @@ -89,6 +129,59 @@ async fn append_batch_commits_multiple_journals_atomically() { assert_eq!(store.dump_journal("default:Order:ord-b").len(), 2); } +#[tokio::test] +async fn append_batch_replays_a_content_bound_claim_without_rescanning_journals() { + let store = SimEventStore::no_faults(96); + let claim = temper_runtime::persistence::PersistenceBatchIdempotency { + persistence_id: "default:Repository:repo-1".to_string(), + idempotency_key: "push-1".to_string(), + intent_hash: "intent-a".to_string(), + }; + let appends = vec![ + PersistenceAppend { + persistence_id: "default:Repository:repo-1".to_string(), + expected_sequence: 0, + events: vec![test_envelope(0, "CompositeEvent")], + key_rows: Vec::new(), + reconcile_keys: false, + key_set_signature: None, + snapshot_source: Default::default(), + batch_idempotency: Some(claim.clone()), + }, + PersistenceAppend { + persistence_id: "default:Commit:commit-1".to_string(), + expected_sequence: 0, + events: vec![test_envelope(0, "Create")], + key_rows: Vec::new(), + reconcile_keys: false, + key_set_signature: None, + snapshot_source: Default::default(), + batch_idempotency: None, + }, + ]; + store.append_batch(&appends).await.expect("first batch"); + + let retry = store + .append_batch(&appends) + .await + .expect("committed retry must be a no-op before stale sequence checks"); + assert!(retry.iter().all(|result| result.batch_already_applied)); + assert_eq!(store.dump_journal("default:Repository:repo-1").len(), 1); + assert_eq!(store.dump_journal("default:Commit:commit-1").len(), 1); + + let mut conflicting = appends; + conflicting[0] + .batch_idempotency + .as_mut() + .expect("claim") + .intent_hash = "intent-b".to_string(); + let error = store + .append_batch(&conflicting) + .await + .expect_err("same idempotency key with different work must fail"); + assert!(error.to_string().contains("different intent")); +} + #[tokio::test] async fn append_batch_conflict_leaves_all_journals_untouched() { let store = SimEventStore::no_faults(42); @@ -107,11 +200,21 @@ async fn append_batch_conflict_leaves_all_journals_untouched() { persistence_id: "default:Order:ord-new".to_string(), expected_sequence: 0, events: vec![test_envelope(0, "Created")], + key_rows: Vec::new(), + reconcile_keys: false, + key_set_signature: None, + snapshot_source: Default::default(), + batch_idempotency: None, }, PersistenceAppend { persistence_id: "default:Order:ord-existing".to_string(), expected_sequence: 0, events: vec![test_envelope(0, "Submitted")], + key_rows: Vec::new(), + reconcile_keys: false, + key_set_signature: None, + snapshot_source: Default::default(), + batch_idempotency: None, }, ]) .await @@ -198,6 +301,288 @@ async fn snapshot_save_records_history_and_rotates_segments() { assert!(!segments[1].sealed); } +#[tokio::test] +async fn delayed_snapshot_splits_durable_tail_and_equal_rewrite_preserves_topology() { + let store = SimEventStore::no_faults(282); + let pid = "default:Order:delayed-segment-snapshot"; + let events = (1..=10) + .map(|sequence| test_envelope(sequence, "Updated")) + .collect::>(); + + store.append(pid, 0, &events).await.unwrap(); + store.save_snapshot(pid, 5, b"snapshot-a").await.unwrap(); + + let expected = vec![ + SimEventSegment { + segment_index: 0, + start_sequence_nr: 1, + end_sequence_nr: Some(5), + snapshot_sequence: Some(5), + event_count: 5, + sealed: true, + }, + SimEventSegment { + segment_index: 1, + start_sequence_nr: 6, + end_sequence_nr: Some(10), + snapshot_sequence: None, + event_count: 5, + sealed: false, + }, + ]; + assert_eq!( + store.dump_segments(pid), + expected, + "a delayed snapshot must split and retain the already-durable tail" + ); + + let topology_before_rewrite = store.dump_segments(pid); + store.save_snapshot(pid, 5, b"snapshot-b").await.unwrap(); + assert_eq!( + store.dump_segments(pid), + topology_before_rewrite, + "same-sequence source replacement must not rotate segment topology" + ); + assert_eq!(store.snapshot_history_len(pid), 1); + assert_eq!( + store.load_snapshot(pid).await.unwrap(), + Some((5, b"snapshot-b".to_vec())) + ); +} + +#[tokio::test] +async fn snapshot_only_and_batch_appends_keep_journal_segments_contiguous() { + let store = SimEventStore::no_faults(283); + let snapshot_only = "default:Order:snapshot-only-first-journal"; + let snapshot = b"snapshot-only".to_vec(); + store + .save_snapshot(snapshot_only, 5, &snapshot) + .await + .unwrap(); + assert!( + store.dump_segments(snapshot_only).is_empty(), + "a snapshot-only generation must not invent journal segments" + ); + store + .append_with_index_rows( + snapshot_only, + 0, + &[test_envelope(0, "FirstJournalEvent")], + &[], + &[], + IndexReconciliation { + snapshot_source: SnapshotSourceFence::Exact { + sequence_nr: 5, + state: snapshot, + }, + ..IndexReconciliation::default() + }, + ) + .await + .unwrap(); + assert_eq!( + store.dump_segments(snapshot_only), + vec![SimEventSegment { + segment_index: 0, + start_sequence_nr: 1, + end_sequence_nr: Some(1), + snapshot_sequence: None, + event_count: 1, + sealed: false, + }] + ); + + let batched = "default:Order:batch-after-snapshot"; + store + .append(batched, 0, &[test_envelope(0, "Created")]) + .await + .unwrap(); + store + .save_snapshot(batched, 1, b"snapshot-1") + .await + .unwrap(); + store + .append_batch(&[PersistenceAppend { + persistence_id: batched.to_string(), + expected_sequence: 1, + events: vec![test_envelope(0, "Batched")], + key_rows: Vec::new(), + reconcile_keys: false, + key_set_signature: None, + snapshot_source: SnapshotSourceFence::Exact { + sequence_nr: 1, + state: b"snapshot-1".to_vec(), + }, + batch_idempotency: None, + }]) + .await + .unwrap(); + assert_eq!( + store.dump_segments(batched), + vec![ + SimEventSegment { + segment_index: 0, + start_sequence_nr: 1, + end_sequence_nr: Some(1), + snapshot_sequence: Some(1), + event_count: 1, + sealed: true, + }, + SimEventSegment { + segment_index: 1, + start_sequence_nr: 2, + end_sequence_nr: Some(2), + snapshot_sequence: None, + event_count: 1, + sealed: false, + }, + ] + ); + + let snapshot_ahead = "default:Order:snapshot-ahead-of-journal"; + store + .append( + snapshot_ahead, + 0, + &[test_envelope(0, "Created"), test_envelope(0, "Updated")], + ) + .await + .unwrap(); + let ahead_snapshot = b"snapshot-5".to_vec(); + store + .save_snapshot(snapshot_ahead, 5, &ahead_snapshot) + .await + .unwrap(); + assert_eq!( + store.dump_segments(snapshot_ahead), + vec![SimEventSegment { + segment_index: 0, + start_sequence_nr: 1, + end_sequence_nr: Some(2), + snapshot_sequence: None, + event_count: 2, + sealed: false, + }], + "snapshot sequence 5 must not rotate topology beyond journal HWM 2" + ); + store + .inner + .lock() + .expect("SimEventStore lock poisoned") + .event_segments + .entry(snapshot_ahead.to_string()) + .or_default() + .push(SimEventSegment { + segment_index: 1, + start_sequence_nr: 6, + end_sequence_nr: None, + snapshot_sequence: None, + event_count: 0, + sealed: false, + }); + store + .append_with_index_rows( + snapshot_ahead, + 2, + &[test_envelope(0, "AfterMigrationSnapshot")], + &[], + &[], + IndexReconciliation { + snapshot_source: SnapshotSourceFence::Exact { + sequence_nr: 5, + state: ahead_snapshot, + }, + ..IndexReconciliation::default() + }, + ) + .await + .unwrap(); + assert_eq!( + store.dump_segments(snapshot_ahead), + vec![SimEventSegment { + segment_index: 0, + start_sequence_nr: 1, + end_sequence_nr: Some(3), + snapshot_sequence: None, + event_count: 3, + sealed: false, + }], + "append must rebuild a legacy future-start segment from journal HWM 2" + ); +} + +#[tokio::test] +async fn materialized_journal_permanently_retires_unchecked_snapshot_generation() { + let store = SimEventStore::no_faults(284); + let persistence_id = "default:Order:materialized-catch-up"; + let migration_snapshot = b"migration-snapshot".to_vec(); + store + .save_snapshot(persistence_id, 5, &migration_snapshot) + .await + .unwrap(); + store + .append_with_index_rows( + persistence_id, + 0, + &[ + materialization_envelope("Order", "materialized-catch-up"), + test_envelope(0, "Changed"), + ], + &[], + &[], + IndexReconciliation { + snapshot_source: SnapshotSourceFence::Exact { + sequence_nr: 5, + state: migration_snapshot, + }, + ..IndexReconciliation::default() + }, + ) + .await + .unwrap(); + store + .append( + persistence_id, + 2, + &[ + test_envelope(0, "Changed"), + test_envelope(0, "Changed"), + test_envelope(0, "Changed"), + test_envelope(0, "Changed"), + ], + ) + .await + .unwrap(); + let topology_before = store.dump_segments(persistence_id); + let history_before = store.snapshot_history_len(persistence_id); + + store + .save_snapshot(persistence_id, 5, b"delayed-unchecked-generation") + .await + .unwrap(); + + assert_eq!(store.load_snapshot(persistence_id).await.unwrap(), None); + assert_eq!(store.snapshot_history_len(persistence_id), history_before); + assert_eq!(store.dump_segments(persistence_id), topology_before); + assert_eq!(store.dump_journal(persistence_id).len(), 6); + + store + .save_snapshot_if_source( + persistence_id, + 6, + b"checked-journal-snapshot", + &SnapshotSourceFence::Absent, + None, + ) + .await + .unwrap(); + assert_eq!( + store.load_snapshot(persistence_id).await.unwrap(), + Some((6, b"checked-journal-snapshot".to_vec())), + "a checked writer may establish the current journal generation's snapshot" + ); +} + #[tokio::test] async fn load_snapshot_returns_none_when_empty() { let store = SimEventStore::no_faults(42); @@ -260,6 +645,150 @@ async fn read_events_from_sequence() { assert_eq!(events[1].sequence_nr, 3); } +#[tokio::test] +async fn read_events_page_applies_cursor_boundary_and_limit() { + let store = SimEventStore::no_faults(43); + let persistence_id = "default:Order:ord-page"; + store + .append( + persistence_id, + 0, + &[ + test_envelope(0, "A"), + test_envelope(0, "B"), + test_envelope(0, "C"), + test_envelope(0, "D"), + test_envelope(0, "E"), + ], + ) + .await + .unwrap(); + + let page = store + .read_events_page(persistence_id, 1, 4, 2) + .await + .unwrap(); + assert_eq!( + page.iter() + .map(|event| event.sequence_nr) + .collect::>(), + vec![2, 3] + ); + let final_page = store + .read_events_page(persistence_id, 3, 4, 8) + .await + .unwrap(); + assert_eq!(final_page.len(), 1); + assert_eq!(final_page[0].sequence_nr, 4); +} + +#[tokio::test] +async fn terminal_tombstone_lookup_is_not_fault_truncated() { + let store = SimEventStore::new( + 43, + SimFaultConfig { + write_failure_prob: 0.0, + concurrency_violation_prob: 0.0, + read_truncation_prob: 1.0, + snapshot_failure_prob: 0.0, + }, + ); + let pid = "default:Order:tombstone-fault"; + let mut deleted = test_envelope(0, "Delete"); + deleted.payload = serde_json::json!({ + "action": "Delete", + "from_status": "Draft", + "to_status": "Deleted" + }); + store + .append(pid, 0, &[test_envelope(0, "Create"), deleted]) + .await + .unwrap(); + + assert_eq!(store.read_events(pid, 0).await.unwrap().len(), 1); + assert_eq!( + store.terminal_tombstone_sequence(pid).await.unwrap(), + Some(2) + ); +} + +#[tokio::test] +async fn snapshot_only_writer_invalidates_inflight_key_coverage() { + let store = SimEventStore::no_faults(44); + let signature = "v4:path"; + let revision = store + .begin_key_index_backfill("default", "Doc", signature) + .await + .unwrap(); + + store + .save_snapshot("default:Doc:snapshot-only", 1, b"snapshot-only") + .await + .unwrap(); + + assert!( + !store + .mark_key_index_backfilled_if_revision("default", "Doc", signature, revision) + .await + .unwrap(), + "an entity created after enumeration must reject stale coverage publication" + ); +} + +#[tokio::test] +async fn failed_snapshot_write_preserves_key_contract_and_coverage() { + let store = SimEventStore::new( + 45, + SimFaultConfig { + write_failure_prob: 0.0, + concurrency_violation_prob: 0.0, + read_truncation_prob: 0.0, + snapshot_failure_prob: 1.0, + }, + ); + let original_signature = "v4:path"; + let original_revision = store + .begin_key_index_backfill("default", "Doc", original_signature) + .await + .unwrap(); + store + .mark_key_index_backfilled("default", "Doc", original_signature) + .await + .unwrap(); + + let result = store + .save_snapshot_if_source( + "default:Doc:failed-snapshot", + 1, + b"snapshot", + &SnapshotSourceFence::Unchecked, + Some("v4:slug"), + ) + .await; + + assert!(matches!(result, Err(PersistenceError::Storage(_)))); + assert_eq!( + store + .key_index_reconciliation_revision("default", "Doc") + .await + .unwrap(), + original_revision, + "a failed snapshot must not advance the key-contract revision" + ); + assert_eq!( + store.key_index_backfilled_types("default").await.unwrap(), + vec![("Doc".to_string(), original_signature.to_string())], + "a failed snapshot must not invalidate proven coverage" + ); + assert_eq!( + store + .load_snapshot("default:Doc:failed-snapshot") + .await + .unwrap(), + None + ); +} + #[tokio::test] async fn deterministic_across_seeds() { // Same seed → same behavior (with no faults, behavior is trivially the same) diff --git a/crates/temper-store-sim/src/tests/key_index.rs b/crates/temper-store-sim/src/tests/key_index.rs new file mode 100644 index 000000000..008e89c62 --- /dev/null +++ b/crates/temper-store-sim/src/tests/key_index.rs @@ -0,0 +1,567 @@ +use super::*; +use temper_runtime::persistence::{ + EntityKeyRow, KeyIndexBackfillFence, SnapshotBackfillFence, encode_activated_key_contract, +}; + +#[tokio::test] +async fn activation_epoch_rejects_delayed_a_writer_after_a_is_reactivated() { + let store = SimEventStore::no_faults(296); + let persistence_id = "default:Doc:delayed-old-writer"; + let old_signature = "v3|4:path[4:Path]"; + let empty_signature = "v3"; + let old_row = EntityKeyRow { + key_name: "path".to_string(), + key_hash: "released-path".to_string(), + }; + + let old_epoch = store + .activate_key_index_contract("default", "Doc", old_signature, false) + .await + .expect("activate original contract"); + store + .mark_key_index_backfilled("default", "Doc", old_signature) + .await + .expect("publish original contract readiness"); + let old_writer_contract = encode_activated_key_contract(old_signature, old_epoch); + store + .append_with_index_rows( + persistence_id, + 0, + &[test_envelope(0, "Created")], + std::slice::from_ref(&old_row), + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(old_writer_contract.clone()), + ..Default::default() + }, + ) + .await + .expect("seed original owner"); + + store + .activate_key_index_contract("default", "Doc", empty_signature, true) + .await + .expect("activate empty contract and purge ownership"); + let current_epoch = store + .activate_key_index_contract("default", "Doc", old_signature, false) + .await + .expect("reactivate original signature at a new epoch"); + let delayed = store + .append_with_index_rows( + persistence_id, + 1, + &[test_envelope(0, "UpdatedByOldTable")], + std::slice::from_ref(&old_row), + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(old_writer_contract), + ..Default::default() + }, + ) + .await; + assert!(matches!( + delayed, + Err(PersistenceError::KeyContractActivationStale { + activated_epoch, + attempted_epoch: Some(attempted_epoch), + }) if activated_epoch == current_epoch && attempted_epoch == old_epoch + )); + assert_eq!(store.dump_journal(persistence_id).len(), 1); + assert_eq!( + store + .lookup_by_key("default", "Doc", "path", &old_row.key_hash) + .await + .expect("lookup released ownership"), + None, + "the rejected old writer must not reinsert a purged claim" + ); +} + +#[tokio::test] +async fn failed_spec_persistence_before_activation_preserves_old_writer_contract() { + let store = SimEventStore::no_faults(297); + let persistence_id = "default:Doc:crash-cut-owner"; + let old_signature = "v3|4:path[4:Path]"; + let old_row = EntityKeyRow { + key_name: "path".to_string(), + key_hash: "crash-cut-path".to_string(), + }; + + let old_epoch = store + .activate_key_index_contract("default", "Doc", old_signature, false) + .await + .expect("activate original contract"); + store + .mark_key_index_backfilled("default", "Doc", old_signature) + .await + .expect("publish original contract readiness"); + let old_writer_contract = encode_activated_key_contract(old_signature, old_epoch); + store + .append_with_index_rows( + persistence_id, + 0, + &[test_envelope(0, "Created")], + std::slice::from_ref(&old_row), + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(old_writer_contract.clone()), + ..Default::default() + }, + ) + .await + .expect("seed original owner"); + store + .mark_key_index_backfilled("default", "Doc", old_signature) + .await + .expect("publish original coverage"); + + // Persist-before-activate rollout means a failed durable spec write never + // touches the live contract. The old table must remain writable after the + // failed install returns; activation is attempted only after persistence. + let failed_persistence: Result<(), &str> = Err("injected durable spec failure"); + assert!(failed_persistence.is_err()); + store + .append_with_index_rows( + persistence_id, + 1, + &[test_envelope(0, "OldSpecStillLive")], + std::slice::from_ref(&old_row), + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(old_writer_contract), + ..Default::default() + }, + ) + .await + .expect("old writer remains accepted after failed persistence"); + + assert!( + store + .key_index_backfilled_types("default") + .await + .expect("read post-crash coverage") + == vec![("Doc".to_string(), old_signature.to_string())], + "failed persistence must preserve the original coverage proof" + ); + assert_eq!( + store + .lookup_by_key("default", "Doc", "path", &old_row.key_hash) + .await + .expect("read post-crash ownership"), + Some("crash-cut-owner".to_string()), + "failed persistence must not purge the original ownership row" + ); +} + +#[tokio::test] +async fn append_batch_reconciles_keys_and_rolls_back_conflicting_claims() { + let store = SimEventStore::no_faults(42); + let owner_pid = "default:Doc:doc-owner"; + let claimant_pid = "default:Doc:doc-claimant"; + let claimed_key = EntityKeyRow { + key_name: "path".to_string(), + key_hash: "shared-path".to_string(), + }; + + store + .append_with_keys( + owner_pid, + 0, + &[test_envelope(0, "Created")], + std::slice::from_ref(&claimed_key), + ) + .await + .unwrap(); + + store + .append_batch(&[ + PersistenceAppend { + persistence_id: owner_pid.to_string(), + expected_sequence: 1, + events: vec![test_envelope(0, "Deleted")], + key_rows: Vec::new(), + reconcile_keys: true, + key_set_signature: Some("v3:path".to_string()), + snapshot_source: Default::default(), + batch_idempotency: None, + }, + PersistenceAppend { + persistence_id: claimant_pid.to_string(), + expected_sequence: 0, + events: vec![test_envelope(0, "Created")], + key_rows: vec![claimed_key.clone()], + reconcile_keys: true, + key_set_signature: Some("v3:path".to_string()), + snapshot_source: Default::default(), + batch_idempotency: None, + }, + ]) + .await + .expect("one atomic batch may release and reclaim the same key"); + assert_eq!( + store + .lookup_by_key("default", "Doc", "path", &claimed_key.key_hash) + .await + .unwrap(), + Some("doc-claimant".to_string()) + ); + assert_eq!(store.dump_journal(owner_pid).len(), 2); + assert_eq!(store.dump_journal(claimant_pid).len(), 1); + + let unrelated_key = EntityKeyRow { + key_name: "path".to_string(), + key_hash: "unrelated-path".to_string(), + }; + let rejected = store + .append_batch(&[ + PersistenceAppend { + persistence_id: "default:Doc:doc-unrelated".to_string(), + expected_sequence: 0, + events: vec![test_envelope(0, "Created")], + key_rows: vec![unrelated_key.clone()], + reconcile_keys: true, + key_set_signature: Some("v3:path".to_string()), + snapshot_source: Default::default(), + batch_idempotency: None, + }, + PersistenceAppend { + persistence_id: "default:Doc:doc-conflict".to_string(), + expected_sequence: 0, + events: vec![test_envelope(0, "Created")], + key_rows: vec![claimed_key.clone()], + reconcile_keys: true, + key_set_signature: Some("v3:path".to_string()), + snapshot_source: Default::default(), + batch_idempotency: None, + }, + ]) + .await; + assert!( + rejected.is_err(), + "an occupied final key must reject the batch" + ); + assert!( + store.dump_journal("default:Doc:doc-unrelated").is_empty(), + "a later key conflict must roll back an earlier stream" + ); + assert!(store.dump_journal("default:Doc:doc-conflict").is_empty()); + assert_eq!( + store + .lookup_by_key("default", "Doc", "path", &unrelated_key.key_hash) + .await + .unwrap(), + None, + "the rejected batch must not publish any key rows" + ); + assert_eq!( + store + .lookup_by_key("default", "Doc", "path", &claimed_key.key_hash) + .await + .unwrap(), + Some("doc-claimant".to_string()), + "the prior owner must survive a rejected batch" + ); +} + +#[tokio::test] +async fn key_reconciliation_includes_snapshot_and_key_only_owners() { + let store = SimEventStore::no_faults(42); + let orphan = EntityKeyRow { + key_name: "path".to_string(), + key_hash: "orphan-path".to_string(), + }; + let repair_signature = "v3:path"; + let repair_revision = store + .begin_key_index_backfill("default", "Doc", repair_signature) + .await + .expect("begin orphan repair contract"); + store + .backfill_entity_keys( + "default", + "Doc", + "orphan-owner", + 0, + KeyIndexBackfillFence { + key_set_signature: repair_signature, + contract_revision: repair_revision, + expected_journal_sequence: 0, + expected_entity_live: false, + expected_snapshot: None, + }, + &[orphan], + ) + .await + .expect("seed key-only orphan"); + store + .save_snapshot("default:Doc:snapshot-only", 5, b"snapshot-only") + .await + .expect("seed snapshot-only owner"); + let repair_revision = store + .begin_key_index_backfill("default", "Doc", repair_signature) + .await + .expect("refresh repair epoch after snapshot-only writer"); + + assert_eq!( + store + .list_entity_ids_by_type("default", "Doc") + .await + .expect("live durable enumeration"), + vec!["snapshot-only".to_string()], + "snapshot-only state is live while a key-only orphan is not" + ); + assert_eq!( + store + .list_entity_ids_for_key_reconciliation("default", "Doc") + .await + .expect("repair enumeration"), + vec!["orphan-owner".to_string(), "snapshot-only".to_string()], + "exact repair must see snapshot and derived owners without journals" + ); + + let snapshot_key = EntityKeyRow { + key_name: "path".to_string(), + key_hash: "snapshot-path".to_string(), + }; + let stale = store + .backfill_entity_keys( + "default", + "Doc", + "snapshot-only", + 0, + KeyIndexBackfillFence { + key_set_signature: repair_signature, + contract_revision: repair_revision, + expected_journal_sequence: 0, + expected_entity_live: true, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: 5, + state: b"snapshot-only", + }), + }, + std::slice::from_ref(&snapshot_key), + ) + .await; + assert!(matches!( + stale, + Err(PersistenceError::ConcurrencyViolation { + expected: 0, + actual: 5 + }) + )); + store + .backfill_entity_keys( + "default", + "Doc", + "snapshot-only", + 5, + KeyIndexBackfillFence { + key_set_signature: repair_signature, + contract_revision: repair_revision, + expected_journal_sequence: 0, + expected_entity_live: true, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: 5, + state: b"snapshot-only", + }), + }, + std::slice::from_ref(&snapshot_key), + ) + .await + .expect("repair snapshot-only owner at durable sequence"); + assert_eq!( + store + .lookup_by_key("default", "Doc", "path", "snapshot-path") + .await + .expect("snapshot-only lookup"), + Some("snapshot-only".to_string()) + ); +} + +#[tokio::test] +async fn key_reconciliation_pages_stop_at_the_captured_candidate_boundary() { + let store = SimEventStore::no_faults(43); + for entity_id in ["a", "b", "c"] { + store + .save_snapshot(&format!("default:Doc:{entity_id}"), 1, b"seed") + .await + .expect("seed initial repair candidate"); + } + + let boundary = store + .key_reconciliation_boundary("default", "Doc") + .await + .expect("capture repair boundary") + .expect("initial candidates establish a boundary"); + assert_eq!(boundary, "c"); + + let mut cursor = None; + let mut observed = Vec::new(); + for growth_index in 0..8 { + let page = store + .list_key_reconciliation_page("default", "Doc", cursor.as_deref(), &boundary, 1) + .await + .expect("read bounded repair page"); + let Some(candidate) = page.into_iter().next() else { + break; + }; + cursor = Some(candidate.entity_id.clone()); + observed.push(candidate.entity_id); + + store + .save_snapshot( + &format!("default:Doc:z-growing-{growth_index}"), + 1, + b"concurrent writer", + ) + .await + .expect("grow the candidate set after boundary capture"); + } + + assert_eq!( + observed, + ["a", "b", "c"], + "concurrent writers beyond the captured terminal ID must not extend repair paging" + ); +} + +#[tokio::test] +async fn journal_source_fence_rejects_equal_sequence_snapshot_repair() { + let store = SimEventStore::no_faults(42); + let persistence_id = "default:Doc:source-aba"; + let repair_signature = "v4:path"; + let snapshot_key = EntityKeyRow { + key_name: "path".to_string(), + key_hash: "snapshot-path".to_string(), + }; + let journal_key = EntityKeyRow { + key_name: "path".to_string(), + key_hash: "journal-path".to_string(), + }; + + store + .save_snapshot(persistence_id, 1, b"snapshot-only") + .await + .expect("seed snapshot-only generation"); + let repair_revision = store + .begin_key_index_backfill("default", "Doc", repair_signature) + .await + .expect("begin snapshot-derived repair"); + + store + .append_with_index_rows( + persistence_id, + 0, + &[test_envelope(0, "Create")], + std::slice::from_ref(&journal_key), + &[], + IndexReconciliation { + keys: true, + key_set_signature: Some(repair_signature.to_string()), + vectors: false, + snapshot_source: Default::default(), + }, + ) + .await + .expect("replace snapshot-only source with equal-sequence journal state"); + + let stale = store + .backfill_entity_keys( + "default", + "Doc", + "source-aba", + 1, + KeyIndexBackfillFence { + key_set_signature: repair_signature, + contract_revision: repair_revision, + expected_journal_sequence: 0, + expected_entity_live: true, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: 1, + state: b"snapshot-only", + }), + }, + std::slice::from_ref(&snapshot_key), + ) + .await; + assert!(matches!( + stale, + Err(PersistenceError::JournalBoundaryChanged { + expected: 0, + actual: 1, + }) + )); + assert_eq!( + store + .lookup_by_key("default", "Doc", "path", &journal_key.key_hash) + .await + .expect("lookup current journal ownership"), + Some("source-aba".to_string()) + ); + assert_eq!( + store + .lookup_by_key("default", "Doc", "path", &snapshot_key.key_hash) + .await + .expect("lookup rejected snapshot ownership"), + None + ); +} + +#[tokio::test] +async fn nonempty_journal_generation_outranks_an_ahead_legacy_snapshot() { + let store = SimEventStore::no_faults(43); + let persistence_id = "default:Doc:journal-generation"; + let repair_signature = "v4:path"; + let journal_key = EntityKeyRow { + key_name: "path".to_string(), + key_hash: "journal-generation-path".to_string(), + }; + + store + .save_snapshot(persistence_id, 5, b"legacy-snapshot-five") + .await + .expect("seed ahead legacy snapshot"); + store + .append( + persistence_id, + 0, + &[test_envelope(0, "Create"), test_envelope(0, "Update")], + ) + .await + .expect("seed lower-coordinate journal generation"); + let repair_revision = store + .begin_key_index_backfill("default", "Doc", repair_signature) + .await + .expect("begin journal-derived repair"); + + store + .backfill_entity_keys( + "default", + "Doc", + "journal-generation", + 2, + KeyIndexBackfillFence { + key_set_signature: repair_signature, + contract_revision: repair_revision, + expected_journal_sequence: 2, + expected_entity_live: true, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: 5, + state: b"legacy-snapshot-five", + }), + }, + std::slice::from_ref(&journal_key), + ) + .await + .expect("journal generation owns the repair coordinate"); + let owner = store + .lookup_by_key_with_sequence("default", "Doc", "path", &journal_key.key_hash) + .await + .expect("lookup journal-generation owner") + .expect("journal-generation key row"); + assert_eq!(owner.entity_id, "journal-generation"); + assert_eq!(owner.sequence_nr, 2); +} diff --git a/crates/temper-store-sim/tests/snapshot_coverage_invalidation.rs b/crates/temper-store-sim/tests/snapshot_coverage_invalidation.rs new file mode 100644 index 000000000..92a27e94f --- /dev/null +++ b/crates/temper-store-sim/tests/snapshot_coverage_invalidation.rs @@ -0,0 +1,141 @@ +//! Regression for snapshot baseline changes after key coverage publication. + +use temper_runtime::persistence::{EventMetadata, EventStore, PersistenceEnvelope}; +use temper_store_sim::SimEventStore; + +#[tokio::test] +async fn older_snapshot_writer_cannot_regress_current_generation_or_coverage() { + let store = SimEventStore::no_faults(277); + let tenant = "default"; + let entity_type = "Doc"; + let persistence_id = format!("{tenant}:{entity_type}:snapshot-writer-race"); + let signature = "v4:path"; + store + .save_snapshot(&persistence_id, 10, b"newer") + .await + .expect("commit newer snapshot writer"); + let revision = store + .begin_key_index_backfill(tenant, entity_type, signature) + .await + .expect("begin coverage epoch"); + assert!( + store + .mark_key_index_backfilled_if_revision(tenant, entity_type, signature, revision) + .await + .expect("publish coverage") + ); + + store + .save_snapshot(&persistence_id, 5, b"delayed-older") + .await + .expect("complete delayed older snapshot writer"); + + assert_eq!( + store + .load_snapshot(&persistence_id) + .await + .expect("load current snapshot"), + Some((10, b"newer".to_vec())), + "a delayed writer must not replace a newer authoritative snapshot" + ); + assert_eq!( + store + .key_index_reconciliation_revision(tenant, entity_type) + .await + .expect("read unchanged coverage epoch"), + revision, + "an ignored older snapshot must not invalidate coverage" + ); + assert_eq!( + store + .key_index_backfilled_types(tenant) + .await + .expect("read preserved coverage watermark"), + vec![(entity_type.to_string(), signature.to_string())] + ); +} + +#[tokio::test] +async fn same_sequence_snapshot_rewrite_invalidates_published_key_coverage() { + let store = SimEventStore::no_faults(273); + let tenant = "default"; + let entity_type = "Doc"; + let persistence_id = format!("{tenant}:{entity_type}:snapshot-rewrite"); + let signature = "v4:path"; + store + .append( + &persistence_id, + 0, + &[PersistenceEnvelope { + sequence_nr: 0, + event_type: "Create".to_string(), + payload: serde_json::json!({}), + metadata: EventMetadata { + event_id: uuid::Uuid::nil(), + causation_id: uuid::Uuid::nil(), + correlation_id: uuid::Uuid::nil(), + timestamp: chrono::DateTime::UNIX_EPOCH, + actor_id: persistence_id.clone(), + }, + }], + ) + .await + .expect("seed journal high-water"); + store + .save_snapshot(&persistence_id, 1, b"before") + .await + .expect("seed captured snapshot baseline"); + let revision = store + .begin_key_index_backfill(tenant, entity_type, signature) + .await + .expect("begin coverage epoch"); + assert!( + store + .mark_key_index_backfilled_if_revision(tenant, entity_type, signature, revision) + .await + .expect("publish coverage") + ); + + store + .save_snapshot(&persistence_id, 1, b"before") + .await + .expect("repeat identical snapshot write"); + assert_eq!( + store + .key_index_reconciliation_revision(tenant, entity_type) + .await + .expect("read unchanged coverage epoch"), + revision, + "identical snapshot bytes and sequence must not churn the coverage epoch" + ); + assert_eq!( + store + .key_index_backfilled_types(tenant) + .await + .expect("read preserved coverage watermark"), + vec![(entity_type.to_string(), signature.to_string())], + "identical snapshot writes must preserve published coverage" + ); + + store + .save_snapshot(&persistence_id, 1, b"after") + .await + .expect("rewrite snapshot bytes at the journal high-water"); + + assert!( + store + .key_index_reconciliation_revision(tenant, entity_type) + .await + .expect("read current coverage epoch") + > revision, + "changed snapshot baseline bytes must invalidate the published coverage epoch" + ); + assert!( + store + .key_index_backfilled_types(tenant) + .await + .expect("read coverage watermarks") + .is_empty(), + "stale ownership rows must not remain authoritative after a snapshot rewrite" + ); +} diff --git a/crates/temper-store-turso/src/lib.rs b/crates/temper-store-turso/src/lib.rs index 231b54209..7a2872957 100644 --- a/crates/temper-store-turso/src/lib.rs +++ b/crates/temper-store-turso/src/lib.rs @@ -73,9 +73,10 @@ pub use metrics::init_metrics; pub use router::{TenantRegistryRow, TenantStoreRouter, TenantUserRow}; pub use store::{ ActionStats, AgentSummary, DesignTimeEventRow, EvolutionRecordRow, FeatureRequestRow, - PolicyDenialPatternRow, PolicyRow, PublishedArtifactRow, PublishedArtifactUpsert, - QueryProjectionUpsert, TursoBlobRow, TursoEventStore, TursoInstalledAppRow, - TursoQueryProjectionRow, TursoSpecRow, TursoTenantConstraintRow, TursoTrajectoryRow, - TursoWasmInvocationRow, TursoWasmModuleMetadataRow, TursoWasmModuleRow, UnmetIntentAggRow, + PolicyDenialPatternRow, PolicyGenerationWrite, PolicyRow, PublishedArtifactRow, + PublishedArtifactUpsert, QueryProjectionUpsert, TursoBlobRow, TursoEventStore, + TursoInstalledAppRow, TursoQueryProjectionRow, TursoSpecRow, TursoTenantConstraintRow, + TursoTrajectoryRow, TursoWasmInvocationRow, TursoWasmModuleMetadataRow, TursoWasmModuleRow, + UnmetIntentAggRow, ots::{OtsQueuedTrajectoryRow, OtsTrajectoryParams, OtsTrajectoryRow}, }; diff --git a/crates/temper-store-turso/src/router.rs b/crates/temper-store-turso/src/router.rs index 3c7e1fe01..58ac52565 100644 --- a/crates/temper-store-turso/src/router.rs +++ b/crates/temper-store-turso/src/router.rs @@ -15,7 +15,8 @@ use tokio::sync::RwLock; use tracing::{info, instrument, warn}; use temper_runtime::persistence::{ - EventStore, PersistenceAppend, PersistenceAppendResult, PersistenceEnvelope, PersistenceError, + EventStore, IndexReconciliation, JournalBoundary, PersistenceAppend, PersistenceAppendResult, + PersistenceBatchIdempotency, PersistenceEnvelope, PersistenceError, SnapshotSourceFence, storage_error, }; use temper_runtime::tenant::parse_persistence_id_parts; @@ -645,6 +646,16 @@ impl TenantStoreRouter { /// `EventStore` implementation that routes by tenant extracted from `persistence_id`. impl EventStore for TenantStoreRouter { + async fn batch_idempotency_committed( + &self, + claim: &PersistenceBatchIdempotency, + ) -> Result { + let (tenant, _, _) = + parse_persistence_id_parts(&claim.persistence_id).map_err(PersistenceError::Storage)?; + let store = self.store_for_tenant(tenant).await?; + store.batch_idempotency_committed(claim).await + } + #[instrument(skip_all, fields(persistence_id, otel.name = "router.append"))] async fn append( &self, @@ -695,6 +706,33 @@ impl EventStore for TenantStoreRouter { store.read_events(persistence_id, from_sequence).await } + #[instrument(skip_all, fields(persistence_id, otel.name = "router.read_events_page"))] + async fn read_events_page( + &self, + persistence_id: &str, + from_sequence: u64, + through_sequence: u64, + limit: usize, + ) -> Result, PersistenceError> { + let (tenant, _, _) = + parse_persistence_id_parts(persistence_id).map_err(PersistenceError::Storage)?; + let store = self.store_for_tenant(tenant).await?; + store + .read_events_page(persistence_id, from_sequence, through_sequence, limit) + .await + } + + #[instrument(skip_all, fields(persistence_id, otel.name = "router.journal_boundary"))] + async fn journal_boundary( + &self, + persistence_id: &str, + ) -> Result { + let (tenant, _, _) = + parse_persistence_id_parts(persistence_id).map_err(PersistenceError::Storage)?; + let store = self.store_for_tenant(tenant).await?; + store.journal_boundary(persistence_id).await + } + #[instrument(skip_all, fields(persistence_id, otel.name = "router.save_snapshot"))] async fn save_snapshot( &self, @@ -710,6 +748,23 @@ impl EventStore for TenantStoreRouter { .await } + #[instrument(skip_all, fields(persistence_id, otel.name = "router.save_snapshot_if_source"))] + async fn save_snapshot_if_source( + &self, + persistence_id: &str, + sequence_nr: u64, + snapshot: &[u8], + source: &SnapshotSourceFence, + key_contract: Option<&str>, + ) -> Result<(), PersistenceError> { + let (tenant, _, _) = + parse_persistence_id_parts(persistence_id).map_err(PersistenceError::Storage)?; + let store = self.store_for_tenant(tenant).await?; + store + .save_snapshot_if_source(persistence_id, sequence_nr, snapshot, source, key_contract) + .await + } + #[instrument(skip_all, fields(persistence_id, otel.name = "router.load_snapshot"))] async fn load_snapshot( &self, @@ -751,7 +806,7 @@ impl EventStore for TenantStoreRouter { events: &[PersistenceEnvelope], key_rows: &[temper_runtime::persistence::EntityKeyRow], vector_rows: &[temper_runtime::persistence::EntityVectorRow], - reconcile_vectors: bool, + reconciliation: IndexReconciliation, ) -> Result { let (tenant, _, _) = parse_persistence_id_parts(persistence_id).map_err(PersistenceError::Storage)?; @@ -763,7 +818,7 @@ impl EventStore for TenantStoreRouter { events, key_rows, vector_rows, - reconcile_vectors, + reconciliation, ) .await } diff --git a/crates/temper-store-turso/src/schema.rs b/crates/temper-store-turso/src/schema.rs index 1012b2e03..36b64d360 100644 --- a/crates/temper-store-turso/src/schema.rs +++ b/crates/temper-store-turso/src/schema.rs @@ -1,50 +1,27 @@ //! SQLite-compatible schema for the Turso/libSQL event store. +mod event_store; mod query_plane; pub use crate::schema_event_history::{ ALTER_EVENTS_ADD_SEGMENT_INDEX, CREATE_EVENT_SEGMENTS_OPEN_INDEX, CREATE_EVENT_SEGMENTS_TABLE, CREATE_SNAPSHOT_HISTORY_ENTITY_INDEX, CREATE_SNAPSHOT_HISTORY_TABLE, }; +pub use event_store::{ + CREATE_EVENTS_ENTITY_INDEX, CREATE_EVENTS_TABLE, CREATE_PERSISTENCE_BATCH_IDEMPOTENCY_TABLE, + CREATE_SNAPSHOTS_TABLE, +}; pub use query_plane::{ CREATE_ENTITY_CATALOG_STATUS_INDEX, CREATE_ENTITY_CATALOG_TABLE, CREATE_ENTITY_CATALOG_TYPE_INDEX, CREATE_ENTITY_FIELD_INDEX_LOOKUP, CREATE_ENTITY_FIELD_INDEX_STATUS, CREATE_ENTITY_FIELD_INDEX_TABLE, CREATE_ENTITY_KEY_INDEX_ENTITY, CREATE_ENTITY_KEY_INDEX_TABLE, CREATE_ENTITY_VECTOR_INDEX_ENTITY, CREATE_ENTITY_VECTOR_INDEX_PARTITION, - CREATE_ENTITY_VECTOR_INDEX_TABLE, CREATE_VECTOR_INDEX_BACKFILL_WATERMARK, + CREATE_ENTITY_VECTOR_INDEX_TABLE, CREATE_QUERY_PROJECTION_DIRTY_TABLE, + CREATE_QUERY_PROJECTION_DIRTY_TYPE_INDEX, CREATE_VECTOR_INDEX_BACKFILL_WATERMARK, + SEED_QUERY_PROJECTION_DIRTY, }; -pub const CREATE_EVENTS_TABLE: &str = "\ -CREATE TABLE IF NOT EXISTS events ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - tenant TEXT NOT NULL, - entity_type TEXT NOT NULL, - entity_id TEXT NOT NULL, - sequence_nr INTEGER NOT NULL, - segment_index INTEGER NOT NULL DEFAULT 0, - event_type TEXT NOT NULL, - payload TEXT NOT NULL, - metadata TEXT, - created_at TEXT NOT NULL DEFAULT (datetime('now')), - UNIQUE(tenant, entity_type, entity_id, sequence_nr) -);"; - -pub const CREATE_EVENTS_ENTITY_INDEX: &str = "\ -CREATE INDEX IF NOT EXISTS idx_events_entity - ON events(tenant, entity_type, entity_id, sequence_nr);"; - -pub const CREATE_SNAPSHOTS_TABLE: &str = "\ -CREATE TABLE IF NOT EXISTS snapshots ( - tenant TEXT NOT NULL, - entity_type TEXT NOT NULL, - entity_id TEXT NOT NULL, - sequence_nr INTEGER NOT NULL, - snapshot BLOB NOT NULL, - created_at TEXT NOT NULL DEFAULT (datetime('now')), - PRIMARY KEY(tenant, entity_type, entity_id) -);"; - pub const CREATE_SPECS_TABLE: &str = "\ CREATE TABLE IF NOT EXISTS specs ( tenant TEXT NOT NULL, diff --git a/crates/temper-store-turso/src/schema/event_store.rs b/crates/temper-store-turso/src/schema/event_store.rs new file mode 100644 index 000000000..ebf6fd938 --- /dev/null +++ b/crates/temper-store-turso/src/schema/event_store.rs @@ -0,0 +1,38 @@ +pub const CREATE_EVENTS_TABLE: &str = "\ +CREATE TABLE IF NOT EXISTS events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + tenant TEXT NOT NULL, + entity_type TEXT NOT NULL, + entity_id TEXT NOT NULL, + sequence_nr INTEGER NOT NULL, + segment_index INTEGER NOT NULL DEFAULT 0, + event_type TEXT NOT NULL, + payload TEXT NOT NULL, + metadata TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE(tenant, entity_type, entity_id, sequence_nr) +);"; + +pub const CREATE_EVENTS_ENTITY_INDEX: &str = "\ +CREATE INDEX IF NOT EXISTS idx_events_entity + ON events(tenant, entity_type, entity_id, sequence_nr);"; + +pub const CREATE_PERSISTENCE_BATCH_IDEMPOTENCY_TABLE: &str = "\ +CREATE TABLE IF NOT EXISTS persistence_batch_idempotency ( + persistence_id TEXT NOT NULL, + idempotency_key TEXT NOT NULL, + intent_hash TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY (persistence_id, idempotency_key) +);"; + +pub const CREATE_SNAPSHOTS_TABLE: &str = "\ +CREATE TABLE IF NOT EXISTS snapshots ( + tenant TEXT NOT NULL, + entity_type TEXT NOT NULL, + entity_id TEXT NOT NULL, + sequence_nr INTEGER NOT NULL, + snapshot BLOB NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY(tenant, entity_type, entity_id) +);"; diff --git a/crates/temper-store-turso/src/schema/query_plane.rs b/crates/temper-store-turso/src/schema/query_plane.rs index 827e3b8df..e211c569a 100644 --- a/crates/temper-store-turso/src/schema/query_plane.rs +++ b/crates/temper-store-turso/src/schema/query_plane.rs @@ -50,6 +50,36 @@ pub const CREATE_ENTITY_FIELD_INDEX_STATUS: &str = "\ CREATE INDEX IF NOT EXISTS idx_efi_status ON entity_field_index(tenant, entity_type, status);"; +/// Entities whose journal/snapshot source is newer than their query projection. +pub const CREATE_QUERY_PROJECTION_DIRTY_TABLE: &str = "\ +CREATE TABLE IF NOT EXISTS query_projection_dirty ( + tenant TEXT NOT NULL, + entity_type TEXT NOT NULL, + entity_id TEXT NOT NULL, + updated_at TEXT NOT NULL DEFAULT '', + PRIMARY KEY (tenant, entity_type, entity_id) +);"; + +/// Bounded dirty-projection enumeration by tenant and entity type. +pub const CREATE_QUERY_PROJECTION_DIRTY_TYPE_INDEX: &str = "\ +CREATE INDEX IF NOT EXISTS idx_query_projection_dirty_type + ON query_projection_dirty(tenant, entity_type, entity_id);"; + +/// One-time upgrade seed for projections created before the dirty-source ledger. +/// +/// The table-existence check in the migration runner ensures this executes only +/// when the ledger is first introduced. Including the source tables covers +/// missing catalog rows; including the catalog covers stale rows whose source +/// has already been retired. Catalog-only compatibility rows are acknowledged +/// and cleared by exact-source repair without being deleted. +pub const SEED_QUERY_PROJECTION_DIRTY: &str = "\ +INSERT OR IGNORE INTO query_projection_dirty (tenant, entity_type, entity_id) +SELECT tenant, entity_type, entity_id FROM events +UNION +SELECT tenant, entity_type, entity_id FROM snapshots +UNION +SELECT tenant, entity_type, entity_id FROM entity_catalog;"; + /// ADR-0153: declared composite-key index — the negative-existence access path. /// /// One row per (declared key, entity), co-committed with the journal append (unlike diff --git a/crates/temper-store-turso/src/store/entity_listing.rs b/crates/temper-store-turso/src/store/entity_listing.rs index 7ccdf6002..96d1fb3f3 100644 --- a/crates/temper-store-turso/src/store/entity_listing.rs +++ b/crates/temper-store-turso/src/store/entity_listing.rs @@ -32,7 +32,11 @@ impl TursoEventStore { WHERE d.tenant = c.tenant AND d.entity_type = c.entity_type AND d.entity_id = c.entity_id - AND d.event_type = 'Deleted' + AND (CASE WHEN json_type(d.payload, '$.to_status') IS NOT NULL + THEN json_extract(d.payload, '$.to_status') = 'Deleted' + ELSE d.event_type = 'Deleted' + OR json_extract(d.payload, '$.action') = 'Deleted' + END) ) UNION SELECT f.entity_id @@ -45,7 +49,11 @@ impl TursoEventStore { WHERE d.tenant = f.tenant AND d.entity_type = f.entity_type AND d.entity_id = f.entity_id - AND d.event_type = 'Deleted' + AND (CASE WHEN json_type(d.payload, '$.to_status') IS NOT NULL + THEN json_extract(d.payload, '$.to_status') = 'Deleted' + ELSE d.event_type = 'Deleted' + OR json_extract(d.payload, '$.action') = 'Deleted' + END) ) UNION SELECT DISTINCT e.entity_id @@ -58,7 +66,11 @@ impl TursoEventStore { WHERE d.tenant = e.tenant AND d.entity_type = e.entity_type AND d.entity_id = e.entity_id - AND d.event_type = 'Deleted' + AND (CASE WHEN json_type(d.payload, '$.to_status') IS NOT NULL + THEN json_extract(d.payload, '$.to_status') = 'Deleted' + ELSE d.event_type = 'Deleted' + OR json_extract(d.payload, '$.action') = 'Deleted' + END) ) ) ORDER BY entity_id", diff --git a/crates/temper-store-turso/src/store/event_store.rs b/crates/temper-store-turso/src/store/event_store.rs index 84d23fb04..29935320f 100644 --- a/crates/temper-store-turso/src/store/event_store.rs +++ b/crates/temper-store-turso/src/store/event_store.rs @@ -1,11 +1,16 @@ //! [`EventStore`] trait implementation for Turso/libSQL. +mod append; +mod append_batch; +mod source_validation; + use libsql::{TransactionBehavior, Value, params, params_from_iter}; use std::time::Duration; use temper_runtime::persistence::{ - EntityVectorCandidate, EntityVectorRow, EventMetadata, EventStore, PersistenceAppend, - PersistenceAppendResult, PersistenceEnvelope, PersistenceError, pack_f32_le, storage_error, - unpack_f32_le, + EntityVectorCandidate, EntityVectorRow, EventMetadata, EventStore, IndexReconciliation, + JournalBoundary, PersistenceAppend, PersistenceAppendResult, PersistenceBatchIdempotency, + PersistenceEnvelope, PersistenceError, SnapshotSourceFence, is_state_materialization_event_for, + is_state_materialization_payload_for, pack_f32_le, storage_error, unpack_f32_le, }; use temper_runtime::tenant::parse_persistence_id_parts; use tracing::{error, instrument, warn}; @@ -19,18 +24,100 @@ use crate::retry::{is_transient_write_error, retry_delay_ms}; const APPEND_BATCH_INSERT_CHUNK_ROWS: usize = 400; +fn append_retires_snapshot( + expected_sequence: u64, + events: &[PersistenceEnvelope], + snapshot_source: &SnapshotSourceFence, + entity_type: &str, + entity_id: &str, +) -> bool { + expected_sequence == 0 + && matches!(snapshot_source, SnapshotSourceFence::Exact { .. }) + && events + .first() + .is_some_and(|event| is_state_materialization_event_for(event, entity_type, entity_id)) +} + struct PreparedEventInsert { tenant: String, entity_type: String, entity_id: String, sequence_nr: u64, + segment_index: i64, event_type: String, payload_json: String, metadata_json: String, expected_sequence: u64, } +pub(super) async fn mark_query_projection_dirty( + tx: &libsql::Transaction, + tenant: &str, + entity_type: &str, + entity_id: &str, +) -> Result<(), PersistenceError> { + let updated_at = temper_runtime::scheduler::sim_now().to_rfc3339(); + tx.execute( + "INSERT INTO query_projection_dirty (tenant, entity_type, entity_id, updated_at) \ + VALUES (?1, ?2, ?3, ?4) \ + ON CONFLICT(tenant, entity_type, entity_id) \ + DO UPDATE SET updated_at = excluded.updated_at", + params![tenant, entity_type, entity_id, updated_at], + ) + .await + .map_err(storage_error)?; + Ok(()) +} + +pub(super) async fn clear_query_projection_dirty( + tx: &libsql::Transaction, + tenant: &str, + entity_type: &str, + entity_id: &str, +) -> Result<(), PersistenceError> { + tx.execute( + "DELETE FROM query_projection_dirty \ + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", + params![tenant, entity_type, entity_id], + ) + .await + .map_err(storage_error)?; + Ok(()) +} + impl EventStore for TursoEventStore { + async fn batch_idempotency_committed( + &self, + claim: &PersistenceBatchIdempotency, + ) -> Result { + let conn = self.configured_connection().await?; + let mut rows = conn + .query( + "SELECT intent_hash FROM persistence_batch_idempotency + WHERE persistence_id = ?1 AND idempotency_key = ?2", + params![ + claim.persistence_id.as_str(), + claim.idempotency_key.as_str() + ], + ) + .await + .map_err(storage_error)?; + let committed_hash = match rows.next().await.map_err(storage_error)? { + Some(row) => Some(row.get::(0).map_err(storage_error)?), + None => None, + }; + let Some(committed_hash) = committed_hash else { + return Ok(false); + }; + if committed_hash != claim.intent_hash { + return Err(PersistenceError::Storage(format!( + "atomic batch idempotency key '{}' was reused with a different intent", + claim.idempotency_key + ))); + } + Ok(true) + } + #[instrument(skip_all, fields(persistence_id, otel.name = "turso.append"))] async fn append( &self, @@ -38,77 +125,13 @@ impl EventStore for TursoEventStore { expected_sequence: u64, events: &[PersistenceEnvelope], ) -> Result { - if events.is_empty() { - return Ok(expected_sequence); - } - - // Retry transient Hrana BLOCKED / stream errors with backoff (ADR-0056). - // Each attempt is a complete append unit. Single-event appends use an - // atomic conditional insert; multi-event appends open a transaction. - // Event-store's UNIQUE (entity_type, entity_id, sequence_nr) makes - // retries safe — if a prior attempt partially committed before erroring, - // the retry's pre-check detects it as ConcurrencyViolation - // (non-transient, propagates to caller via normal event-store contract). - let attempt_timeout = append_attempt_timeout(); - let total_attempts = append_max_attempts(); - let mut last_err: Option = None; - let bypass_write_gate = events.len() == 1; - for attempt in 0..total_attempts { - if attempt > 0 { - tokio::time::sleep(Duration::from_millis(retry_delay_ms(attempt - 1))).await; - } - let _high_priority_marker = if bypass_write_gate { - Some(self.mark_high_priority_write("turso.append")) - } else { - None - }; - let _write_permit = if bypass_write_gate { - None - } else { - Some( - self.acquire_write_permit("turso.append", WritePriority::High) - .await?, - ) - }; - let attempt_result = tokio::time::timeout( - attempt_timeout, - self.append_inner(persistence_id, expected_sequence, events), - ) - .await - .unwrap_or_else(|_| { - warn!( - persistence_id, - attempt, - timeout_ms = attempt_timeout.as_millis() as u64, - "turso.append attempt timed out" - ); - Err(PersistenceError::Storage(format!( - "turso.append timed out after {}ms", - attempt_timeout.as_millis() - ))) - }); - - match attempt_result { - Ok(seq) => { - if attempt > 0 { - record_turso_write_retry("turso.append", attempt as u64, "succeeded"); - } - return Ok(seq); - } - Err(err) => { - let transient = match &err { - PersistenceError::Storage(msg) => is_transient_write_error(msg), - _ => false, - }; - if !transient { - return Err(err); - } - last_err = Some(err); - } - } - } - record_turso_write_retry("turso.append", total_attempts as u64, "exhausted"); - Err(last_err.expect("retry loop captured at least one error")) + self.append_with_snapshot_source( + persistence_id, + expected_sequence, + events, + &SnapshotSourceFence::Unchecked, + ) + .await } async fn lookup_by_key( @@ -139,14 +162,15 @@ impl EventStore for TursoEventStore { // NOTE (ADR-0153): Turso intentionally does NOT implement `backfill_entity_keys`, // `mark_key_index_backfilled`, or `key_index_backfilled_types` — it keeps the - // no-op/empty trait defaults. Turso never co-commits key rows (it does not override - // `append_with_keys`), so its `entity_key_index` is never maintained on write. A - // store that does not maintain the index live must NEVER become authoritative for - // absence: backfilling or watermarking it would let a keyed miss wrongly read a - // present entity as absent (or serve a stale keyed hit). Postgres (the current - // query-plane backend) co-commits and is authoritative; the sim store does too for - // DST. Giving Turso the keyed oracle requires first implementing live co-commit - // (completing ADR-0153 phase 2 for Turso) — tracked separately. + // no-op/empty trait defaults and reports the default non-authoritative capability. + // Turso never co-commits key rows, so its `entity_key_index` is never maintained + // on write. A store that does not maintain the index live must NEVER become + // authoritative for absence: backfilling or watermarking it would let a keyed + // miss wrongly read a present entity as absent (or serve a stale keyed hit). + // Postgres (the current query-plane backend) co-commits and is authoritative; the + // sim store does too for DST. Giving Turso the keyed oracle requires first + // implementing live co-commit (completing ADR-0153 phase 2 for Turso) — tracked + // separately. // ADR-0155: Turso maintains `entity_vector_index` **write-behind** — the event is // appended first (with retries), then the derived vector rows follow in a separate, @@ -161,12 +185,17 @@ impl EventStore for TursoEventStore { events: &[PersistenceEnvelope], _key_rows: &[temper_runtime::persistence::EntityKeyRow], vector_rows: &[EntityVectorRow], - reconcile_vectors: bool, + reconciliation: IndexReconciliation, ) -> Result { // The journal append is the durable event (keys are not maintained on Turso, // per the note above). let new_seq = self - .append(persistence_id, expected_sequence, events) + .append_with_snapshot_source( + persistence_id, + expected_sequence, + events, + &reconciliation.snapshot_source, + ) .await?; // Write-behind vector maintenance: reconcile the entity's rows (delete stale, // insert current — an empty `vector_rows` purges a deleted/cleared entity), @@ -174,7 +203,7 @@ impl EventStore for TursoEventStore { // transient failure does not silently drop the write. On final exhaustion the // error is logged loudly; the partition then lags until the next backfill // reconcile runs. Only runs when the type declares vector paths. - if reconcile_vectors + if reconciliation.vectors && let Ok((tenant, entity_type, entity_id)) = parse_persistence_id_parts(persistence_id) { let total_attempts = append_max_attempts(); @@ -360,17 +389,21 @@ impl EventStore for TursoEventStore { if appends.is_empty() { return Ok(Vec::new()); } - if let [append] = appends { + if let [append] = appends + && append.batch_idempotency.is_none() + { let sequence_nr = self - .append( + .append_with_snapshot_source( &append.persistence_id, append.expected_sequence, &append.events, + &append.snapshot_source, ) .await?; return Ok(vec![PersistenceAppendResult { persistence_id: append.persistence_id.clone(), sequence_nr, + batch_already_applied: false, }]); } @@ -471,15 +504,142 @@ impl EventStore for TursoEventStore { Ok(out) } + #[instrument(skip_all, fields(persistence_id, otel.name = "turso.read_events_page"))] + async fn read_events_page( + &self, + persistence_id: &str, + from_sequence: u64, + through_sequence: u64, + limit: usize, + ) -> Result, PersistenceError> { + assert!(limit > 0, "event page limit must be positive"); + assert!( + through_sequence >= from_sequence, + "event page boundary must not precede its cursor" + ); + let (tenant, entity_type, entity_id) = + parse_persistence_id_parts(persistence_id).map_err(PersistenceError::Storage)?; + let from_sequence = i64::try_from(from_sequence).map_err(|_| { + PersistenceError::Storage("event page cursor exceeds SQLite integer".to_string()) + })?; + let through_sequence = i64::try_from(through_sequence).map_err(|_| { + PersistenceError::Storage("event page boundary exceeds SQLite integer".to_string()) + })?; + let limit = i64::try_from(limit).map_err(|_| { + PersistenceError::Storage("event page limit exceeds SQLite integer".to_string()) + })?; + let conn = self.configured_connection().await?; + + let mut rows = conn + .query( + "SELECT sequence_nr, event_type, payload, metadata + FROM events + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3 + AND sequence_nr > ?4 AND sequence_nr <= ?5 + ORDER BY sequence_nr ASC + LIMIT ?6", + params![ + tenant, + entity_type, + entity_id, + from_sequence, + through_sequence, + limit + ], + ) + .await + .map_err(storage_error)?; + + let mut out = Vec::with_capacity(limit as usize); + while let Some(row) = rows.next().await.map_err(storage_error)? { + let sequence_nr = row.get::(0).map_err(storage_error)? as u64; + let event_type = row.get::(1).map_err(storage_error)?; + let payload_json = row.get::(2).map_err(storage_error)?; + let metadata_json = row.get::>(3).map_err(storage_error)?; + let payload = serde_json::from_str(&payload_json) + .map_err(|error| PersistenceError::Serialization(error.to_string()))?; + let metadata_json = metadata_json.ok_or_else(|| { + PersistenceError::Serialization("missing event metadata".to_string()) + })?; + let metadata = serde_json::from_str(&metadata_json) + .map_err(|error| PersistenceError::Serialization(error.to_string()))?; + out.push(PersistenceEnvelope { + sequence_nr, + event_type, + payload, + metadata, + }); + } + Ok(out) + } + + #[instrument(skip_all, fields(persistence_id, otel.name = "turso.journal_boundary"))] + async fn journal_boundary( + &self, + persistence_id: &str, + ) -> Result { + let (tenant, entity_type, entity_id) = + parse_persistence_id_parts(persistence_id).map_err(PersistenceError::Storage)?; + let conn = self.configured_connection().await?; + let mut rows = conn + .query( + "SELECT COALESCE(MAX(sequence_nr), 0), + MIN(CASE WHEN + CASE WHEN json_type(payload, '$.to_status') IS NOT NULL + THEN json_extract(payload, '$.to_status') = 'Deleted' + ELSE event_type = 'Deleted' + OR json_extract(payload, '$.action') = 'Deleted' + END + THEN sequence_nr END) + FROM events + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", + params![tenant, entity_type, entity_id], + ) + .await + .map_err(storage_error)?; + let Some(row) = rows.next().await.map_err(storage_error)? else { + return Ok(JournalBoundary::default()); + }; + Ok(JournalBoundary { + latest_sequence: row.get::(0).map_err(storage_error)? as u64, + first_terminal_sequence: row + .get::>(1) + .map_err(storage_error)? + .map(|sequence_nr| sequence_nr as u64), + }) + } + #[instrument(skip_all, fields(persistence_id, otel.name = "turso.save_snapshot"))] async fn save_snapshot( &self, persistence_id: &str, sequence_nr: u64, snapshot: &[u8], + ) -> Result<(), PersistenceError> { + self.save_snapshot_if_source( + persistence_id, + sequence_nr, + snapshot, + &SnapshotSourceFence::Unchecked, + None, + ) + .await + } + + #[instrument(skip_all, fields(persistence_id, otel.name = "turso.save_snapshot_if_source"))] + async fn save_snapshot_if_source( + &self, + persistence_id: &str, + sequence_nr: u64, + snapshot: &[u8], + source: &SnapshotSourceFence, + _key_contract: Option<&str>, ) -> Result<(), PersistenceError> { let (tenant, entity_type, entity_id) = parse_persistence_id_parts(persistence_id).map_err(PersistenceError::Storage)?; + let sequence_nr = i64::try_from(sequence_nr).map_err(|_| { + PersistenceError::Storage("snapshot sequence exceeds SQLite integer".to_string()) + })?; let _write_permit = self .acquire_write_permit("turso.save_snapshot", WritePriority::Low) .await?; @@ -489,6 +649,88 @@ impl EventStore for TursoEventStore { .await .map_err(storage_error)?; + Self::validate_snapshot_source(&tx, tenant, entity_type, entity_id, source).await?; + + let mut journal_generation_rows = tx + .query( + "SELECT MAX(sequence_nr), + (SELECT event_type FROM events + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3 + AND sequence_nr = 1), + (SELECT payload FROM events + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3 + AND sequence_nr = 1) + FROM events + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", + params![tenant, entity_type, entity_id], + ) + .await + .map_err(storage_error)?; + let (journal_hwm, first_event_type, first_event_payload) = match journal_generation_rows + .next() + .await + .map_err(storage_error)? + { + Some(row) => ( + row.get::>(0).map_err(storage_error)?, + row.get::>(1).map_err(storage_error)?, + row.get::>(2).map_err(storage_error)?, + ), + None => (None, None, None), + }; + drop(journal_generation_rows); + let first_is_materialization = first_event_payload + .as_deref() + .and_then(|payload| serde_json::from_str::(payload).ok()) + .is_some_and(|payload| { + is_state_materialization_payload_for( + first_event_type.as_deref().unwrap_or_default(), + &payload, + entity_type, + entity_id, + ) + }); + if matches!(source, SnapshotSourceFence::Unchecked) + && journal_hwm.is_some() + && first_is_materialization + { + tx.commit().await.map_err(storage_error)?; + return Ok(()); + } + + let mut current_rows = tx + .query( + "SELECT sequence_nr, snapshot + FROM snapshots + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", + params![tenant, entity_type, entity_id], + ) + .await + .map_err(storage_error)?; + let current_snapshot = match current_rows.next().await.map_err(storage_error)? { + Some(row) => Some(( + row.get::(0).map_err(storage_error)?, + row.get::>(1).map_err(storage_error)?, + )), + None => None, + }; + drop(current_rows); + let snapshot_is_noop = + current_snapshot + .as_ref() + .is_some_and(|(stored_sequence, stored_snapshot)| { + *stored_sequence > sequence_nr + || (*stored_sequence == sequence_nr + && stored_snapshot.as_slice() == snapshot) + }); + if snapshot_is_noop { + tx.commit().await.map_err(storage_error)?; + return Ok(()); + } + let same_sequence_replacement = current_snapshot + .as_ref() + .is_some_and(|(stored_sequence, _)| *stored_sequence == sequence_nr); + tx.execute( "INSERT INTO snapshots (tenant, entity_type, entity_id, sequence_nr, snapshot) VALUES (?1, ?2, ?3, ?4, ?5) @@ -501,7 +743,7 @@ impl EventStore for TursoEventStore { tenant, entity_type, entity_id, - sequence_nr as i64, + sequence_nr, snapshot.to_vec() ], ) @@ -517,19 +759,61 @@ impl EventStore for TursoEventStore { tenant, entity_type, entity_id, - sequence_nr as i64, + sequence_nr, snapshot.to_vec() ], ) .await .map_err(storage_error)?; + mark_query_projection_dirty(&tx, tenant, entity_type, entity_id).await?; + + if same_sequence_replacement { + tx.commit().await.map_err(storage_error)?; + return Ok(()); + } + + let Some(journal_hwm) = journal_hwm else { + // Snapshots may exist before this store has received any journal event. + // They do not establish event-history boundaries. Remove any topology + // left by the legacy behavior and let the first append create segment 0. + tx.execute( + "DELETE FROM event_segments + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", + params![tenant, entity_type, entity_id], + ) + .await + .map_err(storage_error)?; + tx.commit().await.map_err(storage_error)?; + return Ok(()); + }; + if sequence_nr == 0 || sequence_nr > journal_hwm { + // A snapshot can describe state ahead of this store's journal (for + // example, during migration), or the initial state before sequence 1. + // It remains durable snapshot state, but cannot seal a journal range + // that has not been written here. + tx.commit().await.map_err(storage_error)?; + return Ok(()); + } + let mut segment_rows = tx .query( - "SELECT COALESCE(MAX(segment_index), 0) - FROM events - WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3 AND sequence_nr <= ?4", - params![tenant, entity_type, entity_id, sequence_nr as i64], + "SELECT COALESCE( + (SELECT segment_index + FROM events + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3 + AND sequence_nr <= ?4 + ORDER BY sequence_nr DESC + LIMIT 1), + (SELECT segment_index + FROM event_segments + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3 + AND sealed_at IS NULL + ORDER BY segment_index DESC + LIMIT 1), + 0 + )", + params![tenant, entity_type, entity_id, sequence_nr], ) .await .map_err(storage_error)?; @@ -539,51 +823,119 @@ impl EventStore for TursoEventStore { }; drop(segment_rows); + let mut start_rows = tx + .query( + "SELECT start_sequence_nr + FROM event_segments + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3 + AND segment_index = ?4", + params![tenant, entity_type, entity_id, current_segment], + ) + .await + .map_err(storage_error)?; + let current_start = match start_rows.next().await.map_err(storage_error)? { + Some(row) => row.get::(0).map_err(storage_error)?, + None => 1, + }; + drop(start_rows); + + let next_segment = current_segment.checked_add(1).ok_or_else(|| { + PersistenceError::Storage("event segment index exceeds SQLite integer".to_string()) + })?; + let next_sequence = sequence_nr.checked_add(1).ok_or_else(|| { + PersistenceError::Storage("snapshot sequence exceeds SQLite integer".to_string()) + })?; + + // A delayed snapshot may bisect an already-durable open segment. Move the + // complete tail to its successor before sealing the boundary segment. tx.execute( - "INSERT INTO event_segments - (tenant, entity_type, entity_id, segment_index, start_sequence_nr, end_sequence_nr, snapshot_sequence, event_count, sealed_at) - VALUES (?1, ?2, ?3, ?4, 1, ?5, ?5, ?5, datetime('now')) - ON CONFLICT(tenant, entity_type, entity_id, segment_index) DO NOTHING", - params![ - tenant, - entity_type, - entity_id, - current_segment, - sequence_nr as i64 - ], + "UPDATE events + SET segment_index = ?5 + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3 + AND sequence_nr > ?4", + params![tenant, entity_type, entity_id, sequence_nr, next_segment], ) .await .map_err(storage_error)?; + let mut tail_rows = tx + .query( + "SELECT MAX(sequence_nr) + FROM events + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3 + AND sequence_nr > ?4", + params![tenant, entity_type, entity_id, sequence_nr], + ) + .await + .map_err(storage_error)?; + let tail_end = match tail_rows.next().await.map_err(storage_error)? { + Some(row) => row.get::>(0).map_err(storage_error)?, + None => None, + }; + drop(tail_rows); + + // A newer snapshot supersedes every later open-segment shape. Tail events + // were coalesced above, so stale successor metadata can be rebuilt exactly. tx.execute( - "UPDATE event_segments - SET end_sequence_nr = ?5, - snapshot_sequence = ?5, - sealed_at = datetime('now'), - event_count = MAX(?5 - start_sequence_nr + 1, 0) - WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3 AND segment_index = ?4", + "DELETE FROM event_segments + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3 + AND segment_index > ?4", + params![tenant, entity_type, entity_id, current_segment], + ) + .await + .map_err(storage_error)?; + + let current_event_count = if sequence_nr >= current_start { + sequence_nr - current_start + 1 + } else { + 0 + }; + tx.execute( + "INSERT INTO event_segments + (tenant, entity_type, entity_id, segment_index, start_sequence_nr, + end_sequence_nr, snapshot_sequence, event_count, sealed_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?6, ?7, datetime('now')) + ON CONFLICT(tenant, entity_type, entity_id, segment_index) + DO UPDATE SET + start_sequence_nr = excluded.start_sequence_nr, + end_sequence_nr = excluded.end_sequence_nr, + snapshot_sequence = excluded.snapshot_sequence, + event_count = excluded.event_count, + sealed_at = excluded.sealed_at", params![ tenant, entity_type, entity_id, current_segment, - sequence_nr as i64 + current_start, + sequence_nr, + current_event_count ], ) .await .map_err(storage_error)?; + let tail_event_count = tail_end.map_or(0, |end| end - sequence_nr); tx.execute( "INSERT INTO event_segments - (tenant, entity_type, entity_id, segment_index, start_sequence_nr) - VALUES (?1, ?2, ?3, ?4, ?5) - ON CONFLICT(tenant, entity_type, entity_id, segment_index) DO NOTHING", + (tenant, entity_type, entity_id, segment_index, start_sequence_nr, + end_sequence_nr, snapshot_sequence, event_count, sealed_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, NULL, ?7, NULL) + ON CONFLICT(tenant, entity_type, entity_id, segment_index) + DO UPDATE SET + start_sequence_nr = excluded.start_sequence_nr, + end_sequence_nr = excluded.end_sequence_nr, + snapshot_sequence = NULL, + event_count = excluded.event_count, + sealed_at = NULL", params![ tenant, entity_type, entity_id, - current_segment + 1, - sequence_nr as i64 + 1 + next_segment, + next_sequence, + tail_end, + tail_event_count ], ) .await @@ -640,7 +992,11 @@ impl EventStore for TursoEventStore { WHERE d.tenant = e.tenant AND d.entity_type = e.entity_type AND d.entity_id = e.entity_id - AND d.event_type = 'Deleted' + AND (CASE WHEN json_type(d.payload, '$.to_status') IS NOT NULL + THEN json_extract(d.payload, '$.to_status') = 'Deleted' + ELSE d.event_type = 'Deleted' + OR json_extract(d.payload, '$.action') = 'Deleted' + END) )", params![tenant], ) @@ -691,7 +1047,11 @@ impl EventStore for TursoEventStore { WHERE d.tenant = e.tenant AND d.entity_type = e.entity_type AND d.entity_id = e.entity_id - AND d.event_type = 'Deleted' + AND (CASE WHEN json_type(d.payload, '$.to_status') IS NOT NULL + THEN json_extract(d.payload, '$.to_status') = 'Deleted' + ELSE d.event_type = 'Deleted' + OR json_extract(d.payload, '$.action') = 'Deleted' + END) ) ORDER BY e.entity_type, e.entity_id LIMIT ?3", @@ -720,7 +1080,11 @@ impl EventStore for TursoEventStore { WHERE d.tenant = e.tenant AND d.entity_type = e.entity_type AND d.entity_id = e.entity_id - AND d.event_type = 'Deleted' + AND (CASE WHEN json_type(d.payload, '$.to_status') IS NOT NULL + THEN json_extract(d.payload, '$.to_status') = 'Deleted' + ELSE d.event_type = 'Deleted' + OR json_extract(d.payload, '$.action') = 'Deleted' + END) ) ORDER BY e.entity_type, e.entity_id LIMIT ?2", @@ -738,583 +1102,3 @@ impl EventStore for TursoEventStore { Ok(out) } } - -impl TursoEventStore { - /// List tenants with at least one persisted event. - #[instrument(skip_all, fields(otel.name = "turso.list_event_tenants"))] - pub async fn list_event_tenants(&self) -> Result, PersistenceError> { - let conn = self.configured_connection().await?; - let mut rows = conn - .query("SELECT DISTINCT tenant FROM events ORDER BY tenant", ()) - .await - .map_err(storage_error)?; - - let mut out = Vec::new(); - while let Some(row) = rows.next().await.map_err(storage_error)? { - out.push(row.get::(0).map_err(storage_error)?); - } - Ok(out) - } - - /// List tenants appearing in any tenant-scoped storage table. - #[instrument(skip_all, fields(otel.name = "turso.list_storage_tenants"))] - pub async fn list_storage_tenants(&self) -> Result, PersistenceError> { - let conn = self.configured_connection().await?; - let mut rows = conn - .query( - "SELECT tenant FROM events \ - UNION SELECT tenant FROM event_segments \ - UNION SELECT tenant FROM snapshot_history \ - UNION SELECT tenant FROM specs \ - UNION SELECT tenant FROM trajectories \ - UNION SELECT tenant FROM tenant_constraints \ - UNION SELECT tenant FROM wasm_modules \ - UNION SELECT tenant FROM wasm_invocation_logs \ - UNION SELECT tenant FROM pending_decisions \ - UNION SELECT tenant FROM tenant_policies \ - UNION SELECT tenant FROM policies \ - UNION SELECT tenant_id AS tenant FROM tenant_installed_apps \ - UNION SELECT tenant FROM policy_denial_patterns \ - UNION SELECT tenant FROM tenant_secrets \ - UNION SELECT tenant FROM design_time_events \ - UNION SELECT tenant FROM ots_trajectories \ - UNION SELECT tenant FROM entity_catalog \ - ORDER BY tenant", - (), - ) - .await - .map_err(storage_error)?; - - let mut out = Vec::new(); - while let Some(row) = rows.next().await.map_err(storage_error)? { - let tenant = row.get::(0).map_err(storage_error)?; - if !tenant.trim().is_empty() { - out.push(tenant); - } - } - Ok(out) - } - - /// Single-attempt implementation of [`EventStore::append`]. Callers go - /// through the public `append` which wraps this in retry-with-backoff - /// (ADR-0056). Kept as an inherent `async fn` on the concrete type so the - /// transactional body can borrow `self` cleanly across retries without - /// fighting `FnMut` + future-lifetime rules. - /// - /// Safe to retry after a transient transport failure: the UNIQUE - /// constraint on `events.(entity_type, entity_id, sequence_nr)` means a - /// prior-attempt partial commit is detected as `ConcurrencyViolation`, - /// which the retry layer treats as non-transient and propagates to the - /// caller via the normal event-store contract. - async fn append_inner( - &self, - persistence_id: &str, - expected_sequence: u64, - events: &[PersistenceEnvelope], - ) -> Result { - if events.is_empty() { - return Ok(expected_sequence); - } - - if let [event] = events { - return self - .append_single_event_inner(persistence_id, expected_sequence, event) - .await; - } - - let (tenant, entity_type, entity_id) = - parse_persistence_id_parts(persistence_id).map_err(PersistenceError::Storage)?; - let conn = self.configured_connection().await?; - let tx = conn - .transaction_with_behavior(TransactionBehavior::Immediate) - .await - .map_err(storage_error)?; - - let select_start = std::time::Instant::now(); - let rows_result = tx - .query( - "SELECT COALESCE(MAX(sequence_nr), 0) - FROM events - WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", - params![tenant, entity_type, entity_id], - ) - .await; - record_turso_query_duration( - select_start.elapsed(), - "query", - "transaction", - rows_result.is_ok(), - ); - let mut rows = rows_result.map_err(storage_error)?; - - let current_seq = match rows.next().await.map_err(storage_error)? { - Some(row) => row.get::(0).map_err(storage_error)? as u64, - None => 0, - }; - drop(rows); - - if current_seq != expected_sequence { - tracing::error!( - expected = expected_sequence, - actual = current_seq, - "concurrency violation on append" - ); - let _ = tx.rollback().await; - return Err(PersistenceError::ConcurrencyViolation { - expected: expected_sequence, - actual: current_seq, - }); - } - - let segment_index = { - let mut segment_rows = tx - .query( - "SELECT segment_index - FROM event_segments - WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3 AND sealed_at IS NULL - ORDER BY segment_index DESC - LIMIT 1", - params![tenant, entity_type, entity_id], - ) - .await - .map_err(storage_error)?; - if let Some(row) = segment_rows.next().await.map_err(storage_error)? { - row.get::(0).map_err(storage_error)? - } else { - drop(segment_rows); - let mut max_rows = tx - .query( - "SELECT COALESCE(MAX(segment_index), 0) - FROM events - WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", - params![tenant, entity_type, entity_id], - ) - .await - .map_err(storage_error)?; - let idx = match max_rows.next().await.map_err(storage_error)? { - Some(row) => row.get::(0).map_err(storage_error)?, - None => 0, - }; - drop(max_rows); - tx.execute( - "INSERT INTO event_segments - (tenant, entity_type, entity_id, segment_index, start_sequence_nr) - VALUES (?1, ?2, ?3, ?4, ?5) - ON CONFLICT(tenant, entity_type, entity_id, segment_index) DO NOTHING", - params![ - tenant, - entity_type, - entity_id, - idx, - ((current_seq + 1).max(1)) as i64 - ], - ) - .await - .map_err(storage_error)?; - idx - } - }; - - let mut new_seq = expected_sequence; - for event in events { - new_seq += 1; - let payload_json = serde_json::to_string(&event.payload).map_err(|e| { - tracing::error!(error = %e, "failed to serialize event payload"); - PersistenceError::Serialization(e.to_string()) - })?; - let metadata_json = serde_json::to_string(&event.metadata).map_err(|e| { - tracing::error!(error = %e, "failed to serialize event metadata"); - PersistenceError::Serialization(e.to_string()) - })?; - - let insert_start = std::time::Instant::now(); - let insert_result = tx - .execute( - "INSERT INTO events - (tenant, entity_type, entity_id, sequence_nr, segment_index, event_type, payload, metadata) - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", - params![ - tenant, - entity_type, - entity_id, - new_seq as i64, - segment_index, - event.event_type.as_str(), - payload_json, - metadata_json - ], - ) - .await; - record_turso_query_duration( - insert_start.elapsed(), - "execute", - "transaction", - insert_result.is_ok(), - ); - - if let Err(e) = insert_result { - let msg = e.to_string(); - tracing::error!(error = %e, "event insert failed"); - let _ = tx.rollback().await; - if msg.contains("UNIQUE constraint failed") || msg.contains("UNIQUE") { - return Err(PersistenceError::ConcurrencyViolation { - expected: expected_sequence, - actual: new_seq, - }); - } - return Err(PersistenceError::Storage(msg)); - } - } - - if new_seq > expected_sequence { - tx.execute( - "UPDATE event_segments - SET end_sequence_nr = ?5, event_count = MAX(?5 - start_sequence_nr + 1, 0) - WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3 AND segment_index = ?4", - params![ - tenant, - entity_type, - entity_id, - segment_index, - new_seq as i64 - ], - ) - .await - .map_err(storage_error)?; - } - - tx.commit().await.map_err(storage_error)?; - Ok(new_seq) - } - - async fn append_batch_inner( - &self, - appends: &[PersistenceAppend], - ) -> Result, PersistenceError> { - let mut seen = std::collections::BTreeSet::new(); - for append in appends { - if !seen.insert(append.persistence_id.as_str()) { - return Err(PersistenceError::Storage(format!( - "duplicate persistence_id '{}' in append_batch", - append.persistence_id - ))); - } - } - - let conn = self.configured_connection().await?; - let tx = conn - .transaction_with_behavior(TransactionBehavior::Immediate) - .await - .map_err(storage_error)?; - - let mut parsed = Vec::with_capacity(appends.len()); - for append in appends { - let (tenant, entity_type, entity_id) = - parse_persistence_id_parts(&append.persistence_id) - .map_err(PersistenceError::Storage)?; - - if append.expected_sequence == 0 && !append.events.is_empty() { - parsed.push(( - tenant.to_string(), - entity_type.to_string(), - entity_id.to_string(), - )); - continue; - } - - let select_start = std::time::Instant::now(); - let rows_result = tx - .query( - "SELECT COALESCE(MAX(sequence_nr), 0) - FROM events - WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", - params![tenant, entity_type, entity_id], - ) - .await; - record_turso_query_duration( - select_start.elapsed(), - "query", - "transaction", - rows_result.is_ok(), - ); - let mut rows = rows_result.map_err(storage_error)?; - - let current_seq = match rows.next().await.map_err(storage_error)? { - Some(row) => row.get::(0).map_err(storage_error)? as u64, - None => 0, - }; - drop(rows); - - if current_seq != append.expected_sequence { - tracing::error!( - expected = append.expected_sequence, - actual = current_seq, - persistence_id = %append.persistence_id, - "concurrency violation on append_batch" - ); - let _ = tx.rollback().await; - return Err(PersistenceError::ConcurrencyViolation { - expected: append.expected_sequence, - actual: current_seq, - }); - } - parsed.push(( - tenant.to_string(), - entity_type.to_string(), - entity_id.to_string(), - )); - } - - let mut results = Vec::with_capacity(appends.len()); - let mut event_rows = Vec::new(); - for (append, (tenant, entity_type, entity_id)) in appends.iter().zip(parsed.iter()) { - let mut new_seq = append.expected_sequence; - for event in &append.events { - new_seq += 1; - let payload_json = serde_json::to_string(&event.payload).map_err(|e| { - tracing::error!(error = %e, "failed to serialize event payload"); - PersistenceError::Serialization(e.to_string()) - })?; - let metadata_json = serde_json::to_string(&event.metadata).map_err(|e| { - tracing::error!(error = %e, "failed to serialize event metadata"); - PersistenceError::Serialization(e.to_string()) - })?; - - event_rows.push(PreparedEventInsert { - tenant: tenant.clone(), - entity_type: entity_type.clone(), - entity_id: entity_id.clone(), - sequence_nr: new_seq, - event_type: event.event_type.clone(), - payload_json, - metadata_json, - expected_sequence: append.expected_sequence, - }); - } - results.push(PersistenceAppendResult { - persistence_id: append.persistence_id.clone(), - sequence_nr: new_seq, - }); - } - - for chunk in event_rows.chunks(APPEND_BATCH_INSERT_CHUNK_ROWS) { - if chunk.is_empty() { - continue; - } - - let mut insert_sql = String::from( - "INSERT INTO events \ - (tenant, entity_type, entity_id, sequence_nr, event_type, payload, metadata) \ - VALUES ", - ); - let mut insert_values = Vec::with_capacity(chunk.len() * 7); - for (index, row) in chunk.iter().enumerate() { - if index > 0 { - insert_sql.push_str(", "); - } - insert_sql.push_str("(?, ?, ?, ?, ?, ?, ?)"); - insert_values.push(Value::from(row.tenant.clone())); - insert_values.push(Value::from(row.entity_type.clone())); - insert_values.push(Value::from(row.entity_id.clone())); - insert_values.push(Value::from(row.sequence_nr as i64)); - insert_values.push(Value::from(row.event_type.clone())); - insert_values.push(Value::from(row.payload_json.clone())); - insert_values.push(Value::from(row.metadata_json.clone())); - } - - let insert_start = std::time::Instant::now(); - let insert_result = tx - .execute(&insert_sql, params_from_iter(insert_values)) - .await; - record_turso_query_duration( - insert_start.elapsed(), - "execute", - "transaction", - insert_result.is_ok(), - ); - - if let Err(e) = insert_result { - let msg = e.to_string(); - tracing::error!(error = %e, "event batch insert failed"); - let _ = tx.rollback().await; - if msg.contains("UNIQUE constraint failed") || msg.contains("UNIQUE") { - let first = &chunk[0]; - return Err(PersistenceError::ConcurrencyViolation { - expected: first.expected_sequence, - actual: first.sequence_nr, - }); - } - return Err(PersistenceError::Storage(msg)); - } - } - - tx.commit().await.map_err(storage_error)?; - Ok(results) - } - - /// Atomic fast path for the common event-store case: one entity action - /// produces one event. On remote Turso this avoids holding an explicit - /// Hrana transaction across BEGIN/SELECT/INSERT/COMMIT round trips. - async fn append_single_event_inner( - &self, - persistence_id: &str, - expected_sequence: u64, - event: &PersistenceEnvelope, - ) -> Result { - let (tenant, entity_type, entity_id) = - parse_persistence_id_parts(persistence_id).map_err(PersistenceError::Storage)?; - let new_seq = expected_sequence + 1; - let payload_json = serde_json::to_string(&event.payload).map_err(|e| { - tracing::error!(error = %e, "failed to serialize event payload"); - PersistenceError::Serialization(e.to_string()) - })?; - let metadata_json = serde_json::to_string(&event.metadata).map_err(|e| { - tracing::error!(error = %e, "failed to serialize event metadata"); - PersistenceError::Serialization(e.to_string()) - })?; - - let conn = self.configured_connection().await?; - let segment_index = { - let mut rows = conn - .query( - "SELECT segment_index - FROM event_segments - WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3 AND sealed_at IS NULL - ORDER BY segment_index DESC - LIMIT 1", - params![tenant, entity_type, entity_id], - ) - .await - .map_err(storage_error)?; - if let Some(row) = rows.next().await.map_err(storage_error)? { - row.get::(0).map_err(storage_error)? - } else { - drop(rows); - let mut max_rows = conn - .query( - "SELECT COALESCE(MAX(segment_index), 0) - FROM events - WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", - params![tenant, entity_type, entity_id], - ) - .await - .map_err(storage_error)?; - let idx = match max_rows.next().await.map_err(storage_error)? { - Some(row) => row.get::(0).map_err(storage_error)?, - None => 0, - }; - drop(max_rows); - conn.execute( - "INSERT INTO event_segments - (tenant, entity_type, entity_id, segment_index, start_sequence_nr) - VALUES (?1, ?2, ?3, ?4, ?5) - ON CONFLICT(tenant, entity_type, entity_id, segment_index) DO NOTHING", - params![ - tenant, - entity_type, - entity_id, - idx, - ((expected_sequence + 1).max(1)) as i64 - ], - ) - .await - .map_err(storage_error)?; - idx - } - }; - let insert_result = conn - .execute( - "INSERT INTO events - (tenant, entity_type, entity_id, sequence_nr, segment_index, event_type, payload, metadata) - SELECT ?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8 - WHERE ( - SELECT COALESCE(MAX(sequence_nr), 0) - FROM events - WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3 - ) = ?9", - params![ - tenant, - entity_type, - entity_id, - new_seq as i64, - segment_index, - event.event_type.as_str(), - payload_json, - metadata_json, - expected_sequence as i64 - ], - ) - .await; - - let affected = match insert_result { - Ok(affected) => affected, - Err(e) => { - let msg = e.to_string(); - tracing::error!(error = %e, "single event insert failed"); - if msg.contains("UNIQUE constraint failed") || msg.contains("UNIQUE") { - let actual = current_sequence(&conn, tenant, entity_type, entity_id).await?; - return Err(PersistenceError::ConcurrencyViolation { - expected: expected_sequence, - actual, - }); - } - return Err(PersistenceError::Storage(msg)); - } - }; - - if affected == 1 { - conn.execute( - "UPDATE event_segments - SET end_sequence_nr = ?5, event_count = MAX(?5 - start_sequence_nr + 1, 0) - WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3 AND segment_index = ?4", - params![ - tenant, - entity_type, - entity_id, - segment_index, - new_seq as i64 - ], - ) - .await - .map_err(storage_error)?; - return Ok(new_seq); - } - - let actual = current_sequence(&conn, tenant, entity_type, entity_id).await?; - tracing::error!( - expected = expected_sequence, - actual, - affected, - "concurrency violation on single event append" - ); - Err(PersistenceError::ConcurrencyViolation { - expected: expected_sequence, - actual, - }) - } -} - -async fn current_sequence( - conn: &super::instrumentation::InstrumentedConnection, - tenant: &str, - entity_type: &str, - entity_id: &str, -) -> Result { - let mut rows = conn - .query( - "SELECT COALESCE(MAX(sequence_nr), 0) - FROM events - WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", - params![tenant, entity_type, entity_id], - ) - .await - .map_err(storage_error)?; - - match rows.next().await.map_err(storage_error)? { - Some(row) => row - .get::(0) - .map_err(storage_error) - .map(|seq| seq as u64), - None => Ok(0), - } -} diff --git a/crates/temper-store-turso/src/store/event_store/append.rs b/crates/temper-store-turso/src/store/event_store/append.rs new file mode 100644 index 000000000..b7f29a1b2 --- /dev/null +++ b/crates/temper-store-turso/src/store/event_store/append.rs @@ -0,0 +1,202 @@ +//! Source-fenced Turso journal append implementation. + +use super::*; + +impl TursoEventStore { + pub(super) async fn append_with_snapshot_source( + &self, + persistence_id: &str, + expected_sequence: u64, + events: &[PersistenceEnvelope], + snapshot_source: &SnapshotSourceFence, + ) -> Result { + if events.is_empty() { + return Ok(expected_sequence); + } + + let attempt_timeout = append_attempt_timeout(); + let total_attempts = append_max_attempts(); + let mut last_err = None; + let bypass_write_gate = + events.len() == 1 && matches!(snapshot_source, SnapshotSourceFence::Unchecked); + for attempt in 0..total_attempts { + if attempt > 0 { + tokio::time::sleep(Duration::from_millis(retry_delay_ms(attempt - 1))).await; + } + let _high_priority_marker = + bypass_write_gate.then(|| self.mark_high_priority_write("turso.append")); + let _write_permit = if bypass_write_gate { + None + } else { + Some( + self.acquire_write_permit("turso.append", WritePriority::High) + .await?, + ) + }; + let attempt_result = tokio::time::timeout( + attempt_timeout, + self.append_inner(persistence_id, expected_sequence, events, snapshot_source), + ) + .await + .unwrap_or_else(|_| { + warn!( + persistence_id, + attempt, + timeout_ms = attempt_timeout.as_millis() as u64, + "turso.append attempt timed out" + ); + Err(PersistenceError::Storage(format!( + "turso.append timed out after {}ms", + attempt_timeout.as_millis() + ))) + }); + + match attempt_result { + Ok(sequence) => { + if attempt > 0 { + record_turso_write_retry("turso.append", attempt as u64, "succeeded"); + } + return Ok(sequence); + } + Err(error) => { + let transient = matches!( + &error, + PersistenceError::Storage(message) if is_transient_write_error(message) + ); + if !transient { + return Err(error); + } + last_err = Some(error); + } + } + } + record_turso_write_retry("turso.append", total_attempts as u64, "exhausted"); + Err(last_err.expect("retry loop captured at least one error")) + } + + pub(super) async fn append_inner( + &self, + persistence_id: &str, + expected_sequence: u64, + events: &[PersistenceEnvelope], + snapshot_source: &SnapshotSourceFence, + ) -> Result { + if events.is_empty() { + return Ok(expected_sequence); + } + let (tenant, entity_type, entity_id) = + parse_persistence_id_parts(persistence_id).map_err(PersistenceError::Storage)?; + let conn = self.configured_connection().await?; + let tx = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .map_err(storage_error)?; + + let select_start = std::time::Instant::now(); + let rows_result = tx + .query( + "SELECT COALESCE(MAX(sequence_nr), 0) FROM events \ + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", + params![tenant, entity_type, entity_id], + ) + .await; + record_turso_query_duration( + select_start.elapsed(), + "query", + "transaction", + rows_result.is_ok(), + ); + let mut rows = rows_result.map_err(storage_error)?; + let current_sequence = match rows.next().await.map_err(storage_error)? { + Some(row) => row.get::(0).map_err(storage_error)? as u64, + None => 0, + }; + drop(rows); + if current_sequence != expected_sequence { + let _ = tx.rollback().await; + return Err(PersistenceError::ConcurrencyViolation { + expected: expected_sequence, + actual: current_sequence, + }); + } + + Self::validate_snapshot_source(&tx, tenant, entity_type, entity_id, snapshot_source) + .await?; + let segment_index = + Self::prepare_open_segment(&tx, tenant, entity_type, entity_id, current_sequence) + .await?; + + let mut new_sequence = expected_sequence; + for event in events { + new_sequence = new_sequence.checked_add(1).ok_or_else(|| { + PersistenceError::Storage("journal sequence overflow".to_string()) + })?; + let payload_json = serde_json::to_string(&event.payload) + .map_err(|error| PersistenceError::Serialization(error.to_string()))?; + let metadata_json = serde_json::to_string(&event.metadata) + .map_err(|error| PersistenceError::Serialization(error.to_string()))?; + if let Err(error) = tx + .execute( + "INSERT INTO events \ + (tenant, entity_type, entity_id, sequence_nr, segment_index, event_type, payload, metadata) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", + params![ + tenant, + entity_type, + entity_id, + new_sequence as i64, + segment_index, + event.event_type.as_str(), + payload_json, + metadata_json, + ], + ) + .await + { + let message = error.to_string(); + let _ = tx.rollback().await; + if message.contains("UNIQUE") { + return Err(PersistenceError::ConcurrencyViolation { + expected: expected_sequence, + actual: new_sequence, + }); + } + return Err(PersistenceError::Storage(message)); + } + } + + tx.execute( + "UPDATE event_segments \ + SET end_sequence_nr = ?5, event_count = MAX(?5 - start_sequence_nr + 1, 0) \ + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3 AND segment_index = ?4", + params![ + tenant, + entity_type, + entity_id, + segment_index, + new_sequence as i64 + ], + ) + .await + .map_err(storage_error)?; + + if append_retires_snapshot( + expected_sequence, + events, + snapshot_source, + entity_type, + entity_id, + ) { + tx.execute( + "DELETE FROM snapshots \ + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", + params![tenant, entity_type, entity_id], + ) + .await + .map_err(storage_error)?; + } + mark_query_projection_dirty(&tx, tenant, entity_type, entity_id).await?; + tx.commit().await.map_err(storage_error)?; + Ok(new_sequence) + } +} diff --git a/crates/temper-store-turso/src/store/event_store/append_batch.rs b/crates/temper-store-turso/src/store/event_store/append_batch.rs new file mode 100644 index 000000000..685cc2b58 --- /dev/null +++ b/crates/temper-store-turso/src/store/event_store/append_batch.rs @@ -0,0 +1,252 @@ +//! Atomic source-fenced multi-stream Turso appends. + +use super::*; + +impl TursoEventStore { + pub(super) async fn append_batch_inner( + &self, + appends: &[PersistenceAppend], + ) -> Result, PersistenceError> { + let mut seen = std::collections::BTreeSet::new(); + let mut batch_claim = None; + for append in appends { + if !seen.insert(append.persistence_id.as_str()) { + return Err(PersistenceError::Storage(format!( + "duplicate persistence_id '{}' in append_batch", + append.persistence_id + ))); + } + if let Some(claim) = &append.batch_idempotency + && batch_claim.replace(claim).is_some() + { + return Err(PersistenceError::Storage( + "append_batch supplied more than one idempotency claim".to_string(), + )); + } + } + + let conn = self.configured_connection().await?; + let tx = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .map_err(storage_error)?; + + if let Some(claim) = batch_claim { + let mut rows = tx + .query( + "SELECT intent_hash FROM persistence_batch_idempotency \ + WHERE persistence_id = ?1 AND idempotency_key = ?2", + params![ + claim.persistence_id.as_str(), + claim.idempotency_key.as_str() + ], + ) + .await + .map_err(storage_error)?; + if let Some(row) = rows.next().await.map_err(storage_error)? { + let committed_hash = row.get::(0).map_err(storage_error)?; + if committed_hash != claim.intent_hash { + return Err(PersistenceError::Storage(format!( + "atomic batch idempotency key '{}' was reused with a different intent", + claim.idempotency_key + ))); + } + drop(rows); + let mut results = Vec::with_capacity(appends.len()); + for append in appends { + let (tenant, entity_type, entity_id) = + parse_persistence_id_parts(&append.persistence_id) + .map_err(PersistenceError::Storage)?; + let mut sequence_rows = tx + .query( + "SELECT COALESCE(MAX(sequence_nr), 0) FROM events \ + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", + params![tenant, entity_type, entity_id], + ) + .await + .map_err(storage_error)?; + let sequence_nr = match sequence_rows.next().await.map_err(storage_error)? { + Some(row) => row.get::(0).map_err(storage_error)? as u64, + None => 0, + }; + results.push(PersistenceAppendResult { + persistence_id: append.persistence_id.clone(), + sequence_nr, + batch_already_applied: true, + }); + } + return Ok(results); + } + } + + let mut parsed = Vec::with_capacity(appends.len()); + for append in appends { + let (tenant, entity_type, entity_id) = + parse_persistence_id_parts(&append.persistence_id) + .map_err(PersistenceError::Storage)?; + let mut rows = tx + .query( + "SELECT COALESCE(MAX(sequence_nr), 0) FROM events \ + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", + params![tenant, entity_type, entity_id], + ) + .await + .map_err(storage_error)?; + let current_sequence = match rows.next().await.map_err(storage_error)? { + Some(row) => row.get::(0).map_err(storage_error)? as u64, + None => 0, + }; + drop(rows); + if current_sequence != append.expected_sequence { + return Err(PersistenceError::ConcurrencyViolation { + expected: append.expected_sequence, + actual: current_sequence, + }); + } + Self::validate_snapshot_source( + &tx, + tenant, + entity_type, + entity_id, + &append.snapshot_source, + ) + .await?; + let segment_index = + Self::prepare_open_segment(&tx, tenant, entity_type, entity_id, current_sequence) + .await?; + parsed.push(( + tenant.to_string(), + entity_type.to_string(), + entity_id.to_string(), + segment_index, + )); + } + + let mut results = Vec::with_capacity(appends.len()); + let mut event_rows = Vec::new(); + for (append, (tenant, entity_type, entity_id, segment_index)) in + appends.iter().zip(parsed.iter()) + { + let mut new_sequence = append.expected_sequence; + for event in &append.events { + new_sequence = new_sequence.checked_add(1).ok_or_else(|| { + PersistenceError::Storage(format!( + "append_batch sequence overflow for '{}'", + append.persistence_id + )) + })?; + event_rows.push(PreparedEventInsert { + tenant: tenant.clone(), + entity_type: entity_type.clone(), + entity_id: entity_id.clone(), + sequence_nr: new_sequence, + segment_index: *segment_index, + event_type: event.event_type.clone(), + payload_json: serde_json::to_string(&event.payload) + .map_err(|error| PersistenceError::Serialization(error.to_string()))?, + metadata_json: serde_json::to_string(&event.metadata) + .map_err(|error| PersistenceError::Serialization(error.to_string()))?, + expected_sequence: append.expected_sequence, + }); + } + results.push(PersistenceAppendResult { + persistence_id: append.persistence_id.clone(), + sequence_nr: new_sequence, + batch_already_applied: false, + }); + } + + for chunk in event_rows.chunks(APPEND_BATCH_INSERT_CHUNK_ROWS) { + let mut sql = String::from( + "INSERT INTO events \ + (tenant, entity_type, entity_id, sequence_nr, segment_index, event_type, payload, metadata) VALUES ", + ); + let mut values = Vec::with_capacity(chunk.len() * 8); + for (index, row) in chunk.iter().enumerate() { + if index > 0 { + sql.push_str(", "); + } + sql.push_str("(?, ?, ?, ?, ?, ?, ?, ?)"); + values.extend([ + Value::from(row.tenant.clone()), + Value::from(row.entity_type.clone()), + Value::from(row.entity_id.clone()), + Value::from(row.sequence_nr as i64), + Value::from(row.segment_index), + Value::from(row.event_type.clone()), + Value::from(row.payload_json.clone()), + Value::from(row.metadata_json.clone()), + ]); + } + if let Err(error) = tx.execute(&sql, params_from_iter(values)).await { + let message = error.to_string(); + if message.contains("UNIQUE") { + let first = &chunk[0]; + return Err(PersistenceError::ConcurrencyViolation { + expected: first.expected_sequence, + actual: first.sequence_nr, + }); + } + return Err(PersistenceError::Storage(message)); + } + } + + for (append, (tenant, entity_type, entity_id, segment_index)) in + appends.iter().zip(parsed.iter()) + { + let result = results + .iter() + .find(|result| result.persistence_id == append.persistence_id) + .expect("one append result per parsed stream"); + if result.sequence_nr > append.expected_sequence { + tx.execute( + "UPDATE event_segments \ + SET end_sequence_nr = ?5, event_count = MAX(?5 - start_sequence_nr + 1, 0) \ + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3 AND segment_index = ?4", + params![ + tenant.as_str(), + entity_type.as_str(), + entity_id.as_str(), + *segment_index, + result.sequence_nr as i64 + ], + ) + .await + .map_err(storage_error)?; + mark_query_projection_dirty(&tx, tenant, entity_type, entity_id).await?; + } + if append_retires_snapshot( + append.expected_sequence, + &append.events, + &append.snapshot_source, + entity_type, + entity_id, + ) { + tx.execute( + "DELETE FROM snapshots \ + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", + params![tenant.as_str(), entity_type.as_str(), entity_id.as_str()], + ) + .await + .map_err(storage_error)?; + } + } + + if let Some(claim) = batch_claim { + tx.execute( + "INSERT INTO persistence_batch_idempotency \ + (persistence_id, idempotency_key, intent_hash) VALUES (?1, ?2, ?3)", + params![ + claim.persistence_id.as_str(), + claim.idempotency_key.as_str(), + claim.intent_hash.as_str(), + ], + ) + .await + .map_err(storage_error)?; + } + + tx.commit().await.map_err(storage_error)?; + Ok(results) + } +} diff --git a/crates/temper-store-turso/src/store/event_store/source_validation.rs b/crates/temper-store-turso/src/store/event_store/source_validation.rs new file mode 100644 index 000000000..d13b20356 --- /dev/null +++ b/crates/temper-store-turso/src/store/event_store/source_validation.rs @@ -0,0 +1,254 @@ +//! Snapshot-source validation and event-segment selection. + +use super::*; + +impl TursoEventStore { + /// List tenants with at least one persisted event. + #[instrument(skip_all, fields(otel.name = "turso.list_event_tenants"))] + pub async fn list_event_tenants(&self) -> Result, PersistenceError> { + let conn = self.configured_connection().await?; + let mut rows = conn + .query("SELECT DISTINCT tenant FROM events ORDER BY tenant", ()) + .await + .map_err(storage_error)?; + + let mut out = Vec::new(); + while let Some(row) = rows.next().await.map_err(storage_error)? { + out.push(row.get::(0).map_err(storage_error)?); + } + Ok(out) + } + + /// List tenants appearing in any tenant-scoped storage table. + #[instrument(skip_all, fields(otel.name = "turso.list_storage_tenants"))] + pub async fn list_storage_tenants(&self) -> Result, PersistenceError> { + let conn = self.configured_connection().await?; + let mut rows = conn + .query( + "SELECT tenant FROM events \ + UNION SELECT tenant FROM event_segments \ + UNION SELECT tenant FROM snapshot_history \ + UNION SELECT tenant FROM specs \ + UNION SELECT tenant FROM trajectories \ + UNION SELECT tenant FROM tenant_constraints \ + UNION SELECT tenant FROM wasm_modules \ + UNION SELECT tenant FROM wasm_invocation_logs \ + UNION SELECT tenant FROM pending_decisions \ + UNION SELECT tenant FROM tenant_policies \ + UNION SELECT tenant FROM policies \ + UNION SELECT tenant_id AS tenant FROM tenant_installed_apps \ + UNION SELECT tenant FROM policy_denial_patterns \ + UNION SELECT tenant FROM tenant_secrets \ + UNION SELECT tenant FROM design_time_events \ + UNION SELECT tenant FROM ots_trajectories \ + UNION SELECT tenant FROM entity_catalog \ + ORDER BY tenant", + (), + ) + .await + .map_err(storage_error)?; + + let mut out = Vec::new(); + while let Some(row) = rows.next().await.map_err(storage_error)? { + let tenant = row.get::(0).map_err(storage_error)?; + if !tenant.trim().is_empty() { + out.push(tenant); + } + } + Ok(out) + } + + pub(super) async fn validate_snapshot_source( + tx: &libsql::Transaction, + tenant: &str, + entity_type: &str, + entity_id: &str, + source: &SnapshotSourceFence, + ) -> Result<(), PersistenceError> { + if matches!(source, SnapshotSourceFence::Unchecked) { + return Ok(()); + } + let mut rows = tx + .query( + "SELECT sequence_nr, snapshot FROM snapshots \ + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", + params![tenant, entity_type, entity_id], + ) + .await + .map_err(storage_error)?; + let current = match rows.next().await.map_err(storage_error)? { + Some(row) => Some(( + row.get::(0).map_err(storage_error)?, + row.get::>(1).map_err(storage_error)?, + )), + None => None, + }; + let matches = match source { + SnapshotSourceFence::Unchecked => true, + SnapshotSourceFence::Absent => current.is_none(), + SnapshotSourceFence::Exact { sequence_nr, state } => current.is_some_and(|current| { + u64::try_from(current.0).ok() == Some(*sequence_nr) + && current.1.as_slice() == state.as_slice() + }), + }; + if matches { + Ok(()) + } else { + Err(PersistenceError::SnapshotGenerationChanged) + } + } + + /// Return the open segment that must receive the next journal event. + /// Repairs legacy snapshot-only and future-boundary topology in the same + /// transaction that performs the append. + pub(super) async fn prepare_open_segment( + tx: &libsql::Transaction, + tenant: &str, + entity_type: &str, + entity_id: &str, + current_sequence: u64, + ) -> Result { + let current_sequence = i64::try_from(current_sequence).map_err(|_| { + PersistenceError::Storage("journal sequence exceeds SQLite integer".to_string()) + })?; + let next_sequence = current_sequence.checked_add(1).ok_or_else(|| { + PersistenceError::Storage("journal sequence exceeds SQLite integer".to_string()) + })?; + + if current_sequence == 0 { + tx.execute( + "DELETE FROM event_segments \ + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", + params![tenant, entity_type, entity_id], + ) + .await + .map_err(storage_error)?; + tx.execute( + "INSERT INTO event_segments \ + (tenant, entity_type, entity_id, segment_index, start_sequence_nr, \ + end_sequence_nr, snapshot_sequence, event_count, sealed_at) \ + VALUES (?1, ?2, ?3, 0, 1, NULL, NULL, 0, NULL)", + params![tenant, entity_type, entity_id], + ) + .await + .map_err(storage_error)?; + return Ok(0); + } + + let mut open_rows = tx + .query( + "SELECT segment_index FROM event_segments \ + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3 \ + AND sealed_at IS NULL AND start_sequence_nr <= ?4 \ + ORDER BY segment_index DESC LIMIT 1", + params![tenant, entity_type, entity_id, next_sequence], + ) + .await + .map_err(storage_error)?; + if let Some(row) = open_rows.next().await.map_err(storage_error)? { + return row.get::(0).map_err(storage_error); + } + drop(open_rows); + + let mut event_rows = tx + .query( + "SELECT segment_index FROM events \ + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3 \ + ORDER BY sequence_nr DESC LIMIT 1", + params![tenant, entity_type, entity_id], + ) + .await + .map_err(storage_error)?; + let last_event_segment = match event_rows.next().await.map_err(storage_error)? { + Some(row) => row.get::(0).map_err(storage_error)?, + None => 0, + }; + drop(event_rows); + + let mut segment_rows = tx + .query( + "SELECT start_sequence_nr, snapshot_sequence, sealed_at IS NOT NULL \ + FROM event_segments \ + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3 \ + AND segment_index = ?4", + params![tenant, entity_type, entity_id, last_event_segment], + ) + .await + .map_err(storage_error)?; + let segment = match segment_rows.next().await.map_err(storage_error)? { + Some(row) => Some(( + row.get::(0).map_err(storage_error)?, + row.get::>(1).map_err(storage_error)?, + row.get::(2).map_err(storage_error)? != 0, + )), + None => None, + }; + drop(segment_rows); + + let stale_future_boundary = segment.as_ref().is_some_and(|(_, snapshot, sealed)| { + *sealed && snapshot.is_some_and(|snapshot| snapshot > current_sequence) + }); + if segment.as_ref().is_some_and(|(_, _, sealed)| !sealed) || stale_future_boundary { + let start = segment.map_or(1, |(start, _, _)| start); + let count = (current_sequence - start + 1).max(0); + tx.execute( + "DELETE FROM event_segments \ + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3 \ + AND segment_index > ?4", + params![tenant, entity_type, entity_id, last_event_segment], + ) + .await + .map_err(storage_error)?; + tx.execute( + "INSERT INTO event_segments \ + (tenant, entity_type, entity_id, segment_index, start_sequence_nr, \ + end_sequence_nr, snapshot_sequence, event_count, sealed_at) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, NULL, ?7, NULL) \ + ON CONFLICT(tenant, entity_type, entity_id, segment_index) DO UPDATE SET \ + start_sequence_nr = excluded.start_sequence_nr, \ + end_sequence_nr = excluded.end_sequence_nr, \ + snapshot_sequence = NULL, event_count = excluded.event_count, sealed_at = NULL", + params![ + tenant, + entity_type, + entity_id, + last_event_segment, + start, + current_sequence, + count + ], + ) + .await + .map_err(storage_error)?; + return Ok(last_event_segment); + } + + let mut max_rows = tx + .query( + "SELECT MAX(segment_index) FROM event_segments \ + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", + params![tenant, entity_type, entity_id], + ) + .await + .map_err(storage_error)?; + let max_segment = match max_rows.next().await.map_err(storage_error)? { + Some(row) => row + .get::>(0) + .map_err(storage_error)? + .unwrap_or(-1), + None => -1, + }; + let segment_index = max_segment.checked_add(1).ok_or_else(|| { + PersistenceError::Storage("event segment index exceeds SQLite integer".to_string()) + })?; + tx.execute( + "INSERT INTO event_segments \ + (tenant, entity_type, entity_id, segment_index, start_sequence_nr) \ + VALUES (?1, ?2, ?3, ?4, ?5)", + params![tenant, entity_type, entity_id, segment_index, next_sequence], + ) + .await + .map_err(storage_error)?; + Ok(segment_index) + } +} diff --git a/crates/temper-store-turso/src/store/field_index.rs b/crates/temper-store-turso/src/store/field_index.rs index e2f692809..b21f537d5 100644 --- a/crates/temper-store-turso/src/store/field_index.rs +++ b/crates/temper-store-turso/src/store/field_index.rs @@ -5,13 +5,18 @@ //! clauses. This avoids materializing every actor in a collection query //! just to evaluate filters in memory. +mod projection_batch; +mod projection_repair; +mod projection_write; + use libsql::{TransactionBehavior, Value, params, params_from_iter}; use sha2::{Digest, Sha256}; use std::collections::{BTreeMap, BTreeSet}; -use temper_runtime::persistence::{PersistenceError, storage_error}; +use temper_runtime::persistence::{PersistenceError, ProjectionSourceFence, storage_error}; use temper_runtime::scheduler::sim_now; use tracing::instrument; +use super::event_store::{clear_query_projection_dirty, mark_query_projection_dirty}; use super::write_gate::WritePriority; use super::{TursoEventStore, TursoQueryProjectionRow}; @@ -83,492 +88,6 @@ pub struct QueryProjectionUpsert { } impl TursoEventStore { - /// Upsert the durable query-plane projection for a single entity. - /// - /// Maintains both: - /// - `entity_catalog`: one live row per entity - /// - `entity_field_index`: EAV rows for OData filter push-down - #[instrument(skip_all, fields( - otel.name = "turso.upsert_query_projection", - tenant, entity_type, entity_id, - ))] - pub async fn upsert_query_projection( - &self, - tenant: &str, - entity_type: &str, - entity_id: &str, - status: &str, - fields: &serde_json::Value, - sequence_nr: u64, - ) -> Result<(), PersistenceError> { - let state = serde_json::json!({ - "entity_type": entity_type, - "entity_id": entity_id, - "status": status, - "item_count": 0, - "counters": {}, - "booleans": {}, - "lists": {}, - "fields": fields, - "events": [], - "total_event_count": sequence_nr, - "sequence_nr": sequence_nr - }); - self.upsert_query_projection_with_state( - tenant, - entity_type, - entity_id, - status, - fields, - &state, - sequence_nr, - ) - .await - } - - #[expect(clippy::too_many_arguments, reason = "projection upsert boundary")] - pub async fn upsert_query_projection_with_state( - &self, - tenant: &str, - entity_type: &str, - entity_id: &str, - status: &str, - fields: &serde_json::Value, - state: &serde_json::Value, - sequence_nr: u64, - ) -> Result<(), PersistenceError> { - let status = canonical_projection_status(status, state); - let _write_permit = self - .acquire_write_permit("turso.upsert_query_projection", WritePriority::Low) - .await?; - let conn = self.configured_connection().await?; - let new_projection_hash = projection_hash(status, fields); - let fields_json = serde_json::to_string(fields) - .map_err(|error| PersistenceError::Serialization(error.to_string()))?; - let state_json = serde_json::to_string(state) - .map_err(|error| PersistenceError::Serialization(error.to_string()))?; - let updated_at = sim_now().to_rfc3339(); - let sequence_nr = i64::try_from(sequence_nr).unwrap_or(i64::MAX); - - let unchanged_rows = conn - .execute( - "UPDATE entity_catalog \ - SET status = ?4, updated_at = ?5, sequence_nr = ?6, projection_version = 2, fields = ?8, state = ?9 \ - WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3 AND projection_hash = ?7 \ - AND sequence_nr <= ?6", - params![ - tenant, - entity_type, - entity_id, - status, - updated_at.as_str(), - sequence_nr, - new_projection_hash.as_str(), - fields_json.as_str(), - state_json.as_str(), - ], - ) - .await - .map_err(storage_error)?; - - if unchanged_rows > 0 { - return Ok(()); - } - - let tx = conn - .transaction_with_behavior(TransactionBehavior::Immediate) - .await - .map_err(storage_error)?; - - let existing_sequence = { - let mut existing_rows = tx - .query( - "SELECT sequence_nr FROM entity_catalog \ - WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", - params![tenant, entity_type, entity_id], - ) - .await - .map_err(storage_error)?; - if let Some(row) = existing_rows.next().await.map_err(storage_error)? { - Some(row.get::(0).map_err(storage_error)?) - } else { - None - } - }; - if existing_sequence.is_some_and(|existing| existing > sequence_nr) { - tx.commit().await.map_err(storage_error)?; - return Ok(()); - } - - tx.execute( - "INSERT INTO entity_catalog \ - (tenant, entity_type, entity_id, status, fields, state, updated_at, sequence_nr, projection_version, projection_hash) \ - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, 2, ?9) \ - ON CONFLICT(tenant, entity_type, entity_id) DO UPDATE SET \ - status = excluded.status, \ - fields = excluded.fields, \ - state = excluded.state, \ - updated_at = excluded.updated_at, \ - sequence_nr = excluded.sequence_nr, \ - projection_version = excluded.projection_version, \ - projection_hash = excluded.projection_hash", - params![ - tenant, - entity_type, - entity_id, - status, - fields_json.as_str(), - state_json.as_str(), - updated_at.as_str(), - sequence_nr, - new_projection_hash.as_str(), - ], - ) - .await - .map_err(storage_error)?; - - // Delete existing rows for this entity, then re-insert. - // This is simpler than tracking individual field changes and handles - // field removal correctly. - tx.execute( - "DELETE FROM entity_field_index WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", - params![tenant, entity_type, entity_id], - ) - .await - .map_err(storage_error)?; - - let indexed_fields = indexed_projection_fields(status, fields); - if !indexed_fields.is_empty() { - let mut sql = String::from( - "INSERT INTO entity_field_index \ - (tenant, entity_type, entity_id, field_name, field_value, status) VALUES ", - ); - let mut values = Vec::with_capacity(indexed_fields.len() * 6); - - for (index, (field_name, field_value)) in indexed_fields.iter().enumerate() { - if index > 0 { - sql.push_str(", "); - } - sql.push_str("(?, ?, ?, ?, ?, ?)"); - values.push(Value::from(tenant.to_string())); - values.push(Value::from(entity_type.to_string())); - values.push(Value::from(entity_id.to_string())); - values.push(Value::from(field_name.clone())); - values.push( - field_value - .as_ref() - .map(|value| Value::from(value.clone())) - .unwrap_or(Value::Null), - ); - values.push(Value::from(status.to_string())); - } - - tx.execute(&sql, params_from_iter(values)) - .await - .map_err(storage_error)?; - } - - tx.commit().await.map_err(storage_error)?; - Ok(()) - } - - /// Upsert several durable query-plane projections in one transaction. - #[instrument(skip_all, fields( - otel.name = "turso.upsert_query_projections", - tenant, - projection_count = projections.len(), - ))] - pub async fn upsert_query_projections( - &self, - tenant: &str, - projections: &[QueryProjectionUpsert], - ) -> Result<(), PersistenceError> { - if projections.is_empty() { - return Ok(()); - } - - let _write_permit = self - .acquire_write_permit("turso.upsert_query_projections", WritePriority::Low) - .await?; - let conn = self.configured_connection().await?; - let tx = conn - .transaction_with_behavior(TransactionBehavior::Immediate) - .await - .map_err(storage_error)?; - let updated_at = sim_now().to_rfc3339(); - let mut existing = BTreeMap::new(); - let existing_candidates = projections - .iter() - .filter(|projection| !projection.known_new) - .collect::>(); - if !existing_candidates.is_empty() { - let mut existing_sql = String::from( - "SELECT entity_type, entity_id, sequence_nr, projection_hash \ - FROM entity_catalog WHERE tenant = ? AND (", - ); - let mut existing_values = Vec::with_capacity(1 + existing_candidates.len() * 2); - existing_values.push(Value::from(tenant.to_string())); - for (index, projection) in existing_candidates.iter().enumerate() { - if index > 0 { - existing_sql.push_str(" OR "); - } - existing_sql.push_str("(entity_type = ? AND entity_id = ?)"); - existing_values.push(Value::from(projection.entity_type.clone())); - existing_values.push(Value::from(projection.entity_id.clone())); - } - existing_sql.push(')'); - - let mut existing_rows = tx - .query(&existing_sql, params_from_iter(existing_values)) - .await - .map_err(storage_error)?; - while let Some(row) = existing_rows.next().await.map_err(storage_error)? { - let entity_type: String = row.get(0).map_err(storage_error)?; - let entity_id: String = row.get(1).map_err(storage_error)?; - let sequence_nr: i64 = row.get(2).map_err(storage_error)?; - let projection_hash: String = row.get(3).map_err(storage_error)?; - existing.insert((entity_type, entity_id), (sequence_nr, projection_hash)); - } - drop(existing_rows); - } - - struct PreparedProjection { - entity_type: String, - entity_id: String, - status: String, - fields_json: String, - state_json: String, - sequence_nr: i64, - projection_hash: String, - } - - let mut catalog_upserts = Vec::new(); - let mut changed_indexes = Vec::new(); - for projection in projections { - let new_projection_hash = projection_hash(&projection.status, &projection.fields); - let fields_json = serde_json::to_string(&projection.fields) - .map_err(|error| PersistenceError::Serialization(error.to_string()))?; - let state_json = serde_json::to_string(&projection.state) - .map_err(|error| PersistenceError::Serialization(error.to_string()))?; - let sequence_nr = i64::try_from(projection.sequence_nr).unwrap_or(i64::MAX); - let key = (projection.entity_type.clone(), projection.entity_id.clone()); - let existing_row = existing.get(&key); - if existing_row.is_some_and(|(existing_sequence, _)| *existing_sequence > sequence_nr) { - continue; - } - if existing_row.is_none_or(|(_, existing_hash)| existing_hash != &new_projection_hash) { - changed_indexes.push(( - projection.entity_type.clone(), - projection.entity_id.clone(), - projection.status.clone(), - indexed_projection_fields(&projection.status, &projection.indexed_fields), - projection.known_new, - )); - } - catalog_upserts.push(PreparedProjection { - entity_type: projection.entity_type.clone(), - entity_id: projection.entity_id.clone(), - status: projection.status.clone(), - fields_json, - state_json, - sequence_nr, - projection_hash: new_projection_hash, - }); - } - - if !catalog_upserts.is_empty() { - let mut upsert_sql = String::from( - "INSERT INTO entity_catalog \ - (tenant, entity_type, entity_id, status, fields, state, updated_at, sequence_nr, projection_version, projection_hash) \ - VALUES ", - ); - let mut upsert_values = Vec::with_capacity(catalog_upserts.len() * 9); - for (index, projection) in catalog_upserts.iter().enumerate() { - if index > 0 { - upsert_sql.push_str(", "); - } - upsert_sql.push_str("(?, ?, ?, ?, ?, ?, ?, ?, 2, ?)"); - upsert_values.push(Value::from(tenant.to_string())); - upsert_values.push(Value::from(projection.entity_type.clone())); - upsert_values.push(Value::from(projection.entity_id.clone())); - upsert_values.push(Value::from(projection.status.clone())); - upsert_values.push(Value::from(projection.fields_json.clone())); - upsert_values.push(Value::from(projection.state_json.clone())); - upsert_values.push(Value::from(updated_at.clone())); - upsert_values.push(Value::from(projection.sequence_nr)); - upsert_values.push(Value::from(projection.projection_hash.clone())); - } - upsert_sql.push_str( - " ON CONFLICT(tenant, entity_type, entity_id) DO UPDATE SET \ - status = excluded.status, \ - fields = excluded.fields, \ - state = excluded.state, \ - updated_at = excluded.updated_at, \ - sequence_nr = excluded.sequence_nr, \ - projection_version = excluded.projection_version, \ - projection_hash = excluded.projection_hash \ - WHERE entity_catalog.sequence_nr <= excluded.sequence_nr", - ); - tx.execute(&upsert_sql, params_from_iter(upsert_values)) - .await - .map_err(storage_error)?; - } - - if !changed_indexes.is_empty() { - let delete_targets = changed_indexes - .iter() - .filter(|(_, _, _, _, known_new)| !known_new) - .collect::>(); - if !delete_targets.is_empty() { - let mut delete_sql = - String::from("DELETE FROM entity_field_index WHERE tenant = ? AND ("); - let mut delete_values = Vec::with_capacity(1 + delete_targets.len() * 2); - delete_values.push(Value::from(tenant.to_string())); - for (index, (entity_type, entity_id, _, _, _)) in delete_targets.iter().enumerate() - { - if index > 0 { - delete_sql.push_str(" OR "); - } - delete_sql.push_str("(entity_type = ? AND entity_id = ?)"); - delete_values.push(Value::from(entity_type.clone())); - delete_values.push(Value::from(entity_id.clone())); - } - delete_sql.push(')'); - tx.execute(&delete_sql, params_from_iter(delete_values)) - .await - .map_err(storage_error)?; - } - - let indexed_field_count = changed_indexes - .iter() - .map(|(_, _, _, fields, _)| fields.len()) - .sum::(); - if indexed_field_count > 0 { - let mut insert_sql = String::from( - "INSERT INTO entity_field_index \ - (tenant, entity_type, entity_id, field_name, field_value, status) VALUES ", - ); - let mut insert_values = Vec::with_capacity(indexed_field_count * 6); - let mut field_index = 0usize; - for (entity_type, entity_id, status, indexed_fields, _) in &changed_indexes { - for (field_name, field_value) in indexed_fields { - if field_index > 0 { - insert_sql.push_str(", "); - } - insert_sql.push_str("(?, ?, ?, ?, ?, ?)"); - insert_values.push(Value::from(tenant.to_string())); - insert_values.push(Value::from(entity_type.clone())); - insert_values.push(Value::from(entity_id.clone())); - insert_values.push(Value::from(field_name.clone())); - insert_values.push( - field_value - .as_ref() - .map(|value| Value::from(value.clone())) - .unwrap_or(Value::Null), - ); - insert_values.push(Value::from(status.clone())); - field_index += 1; - } - } - - tx.execute(&insert_sql, params_from_iter(insert_values)) - .await - .map_err(storage_error)?; - } - } - - tx.commit().await.map_err(storage_error)?; - Ok(()) - } - - /// Backwards-compatible alias for the old name. - #[instrument(skip_all, fields( - otel.name = "turso.upsert_field_index", - tenant, entity_type, entity_id, - ))] - pub async fn upsert_field_index( - &self, - tenant: &str, - entity_type: &str, - entity_id: &str, - status: &str, - fields: &serde_json::Value, - ) -> Result<(), PersistenceError> { - let state = serde_json::json!({ - "entity_type": entity_type, - "entity_id": entity_id, - "status": status, - "item_count": 0, - "counters": {}, - "booleans": {}, - "lists": {}, - "fields": fields, - "events": [], - "total_event_count": 0, - "sequence_nr": 0 - }); - self.upsert_query_projection_with_state( - tenant, - entity_type, - entity_id, - status, - fields, - &state, - 0, - ) - .await - } - - /// Remove the durable query-plane projection for a single entity. - #[instrument(skip_all, fields( - otel.name = "turso.remove_query_projection", - tenant, entity_type, entity_id, - ))] - pub async fn remove_query_projection( - &self, - tenant: &str, - entity_type: &str, - entity_id: &str, - ) -> Result<(), PersistenceError> { - let _write_permit = self - .acquire_write_permit("turso.remove_query_projection", WritePriority::Low) - .await?; - let conn = self.configured_connection().await?; - let tx = conn - .transaction_with_behavior(TransactionBehavior::Immediate) - .await - .map_err(storage_error)?; - tx.execute( - "DELETE FROM entity_catalog WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", - params![tenant, entity_type, entity_id], - ) - .await - .map_err(storage_error)?; - tx.execute( - "DELETE FROM entity_field_index WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", - params![tenant, entity_type, entity_id], - ) - .await - .map_err(storage_error)?; - tx.commit().await.map_err(storage_error)?; - Ok(()) - } - - /// Backwards-compatible alias for the old name. - #[instrument(skip_all, fields( - otel.name = "turso.remove_field_index", - tenant, entity_type, entity_id, - ))] - pub async fn remove_field_index( - &self, - tenant: &str, - entity_type: &str, - entity_id: &str, - ) -> Result<(), PersistenceError> { - self.remove_query_projection(tenant, entity_type, entity_id) - .await - } - /// Query the field index with a pre-built SQL WHERE clause. /// /// Returns distinct entity IDs matching the filter. The `where_clause` @@ -781,6 +300,34 @@ impl TursoEventStore { Ok(counts) } + /// Return a bounded set of entities requiring exact-source projection repair. + pub async fn dirty_query_projection_entity_ids( + &self, + tenant: &str, + entity_type: &str, + limit: usize, + ) -> Result, PersistenceError> { + let conn = self.configured_connection().await?; + let mut rows = conn + .query( + "SELECT entity_id FROM query_projection_dirty \ + WHERE tenant = ?1 AND entity_type = ?2 \ + ORDER BY entity_id LIMIT ?3", + params![ + tenant, + entity_type, + i64::try_from(limit).unwrap_or(i64::MAX) + ], + ) + .await + .map_err(storage_error)?; + let mut entity_ids = Vec::new(); + while let Some(row) = rows.next().await.map_err(storage_error)? { + entity_ids.push(row.get::(0).map_err(storage_error)?); + } + Ok(entity_ids) + } + /// Batch-load full entity catalog rows for a list of entity IDs. /// /// Returns only rows that exist in the catalog; missing IDs are silently diff --git a/crates/temper-store-turso/src/store/field_index/projection_batch.rs b/crates/temper-store-turso/src/store/field_index/projection_batch.rs new file mode 100644 index 000000000..998d337c8 --- /dev/null +++ b/crates/temper-store-turso/src/store/field_index/projection_batch.rs @@ -0,0 +1,290 @@ +//! Batched durable query-projection writes. + +use super::*; + +impl TursoEventStore { + /// Upsert several durable query-plane projections in one transaction. + #[instrument(skip_all, fields( + otel.name = "turso.upsert_query_projections", + tenant, + projection_count = projections.len(), + ))] + pub async fn upsert_query_projections( + &self, + tenant: &str, + projections: &[QueryProjectionUpsert], + ) -> Result<(), PersistenceError> { + if projections.is_empty() { + return Ok(()); + } + + let _write_permit = self + .acquire_write_permit("turso.upsert_query_projections", WritePriority::Low) + .await?; + let conn = self.configured_connection().await?; + let tx = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .map_err(storage_error)?; + let updated_at = sim_now().to_rfc3339(); + let mut journal_sequences = BTreeMap::new(); + let mut journal_sql = String::from( + "SELECT entity_type, entity_id, MAX(sequence_nr) FROM events \ + WHERE tenant = ? AND (", + ); + let mut journal_values = Vec::with_capacity(1 + projections.len() * 2); + journal_values.push(Value::from(tenant.to_string())); + for (index, projection) in projections.iter().enumerate() { + if index > 0 { + journal_sql.push_str(" OR "); + } + journal_sql.push_str("(entity_type = ? AND entity_id = ?)"); + journal_values.push(Value::from(projection.entity_type.clone())); + journal_values.push(Value::from(projection.entity_id.clone())); + } + journal_sql.push_str(") GROUP BY entity_type, entity_id"); + let mut journal_rows = tx + .query(&journal_sql, params_from_iter(journal_values)) + .await + .map_err(storage_error)?; + while let Some(row) = journal_rows.next().await.map_err(storage_error)? { + let entity_type: String = row.get(0).map_err(storage_error)?; + let entity_id: String = row.get(1).map_err(storage_error)?; + let sequence_nr: i64 = row.get(2).map_err(storage_error)?; + journal_sequences.insert((entity_type, entity_id), sequence_nr); + } + drop(journal_rows); + + let mut snapshot_entities = BTreeSet::new(); + let mut snapshot_sql = + String::from("SELECT entity_type, entity_id FROM snapshots WHERE tenant = ? AND ("); + let mut snapshot_values = Vec::with_capacity(1 + projections.len() * 2); + snapshot_values.push(Value::from(tenant.to_string())); + for (index, projection) in projections.iter().enumerate() { + if index > 0 { + snapshot_sql.push_str(" OR "); + } + snapshot_sql.push_str("(entity_type = ? AND entity_id = ?)"); + snapshot_values.push(Value::from(projection.entity_type.clone())); + snapshot_values.push(Value::from(projection.entity_id.clone())); + } + snapshot_sql.push(')'); + let mut snapshot_rows = tx + .query(&snapshot_sql, params_from_iter(snapshot_values)) + .await + .map_err(storage_error)?; + while let Some(row) = snapshot_rows.next().await.map_err(storage_error)? { + snapshot_entities.insert(( + row.get::(0).map_err(storage_error)?, + row.get::(1).map_err(storage_error)?, + )); + } + drop(snapshot_rows); + + let mut existing = BTreeMap::new(); + let existing_candidates = projections.iter().collect::>(); + if !existing_candidates.is_empty() { + let mut existing_sql = String::from( + "SELECT entity_type, entity_id, sequence_nr, projection_hash \ + FROM entity_catalog WHERE tenant = ? AND (", + ); + let mut existing_values = Vec::with_capacity(1 + existing_candidates.len() * 2); + existing_values.push(Value::from(tenant.to_string())); + for (index, projection) in existing_candidates.iter().enumerate() { + if index > 0 { + existing_sql.push_str(" OR "); + } + existing_sql.push_str("(entity_type = ? AND entity_id = ?)"); + existing_values.push(Value::from(projection.entity_type.clone())); + existing_values.push(Value::from(projection.entity_id.clone())); + } + existing_sql.push(')'); + + let mut existing_rows = tx + .query(&existing_sql, params_from_iter(existing_values)) + .await + .map_err(storage_error)?; + while let Some(row) = existing_rows.next().await.map_err(storage_error)? { + let entity_type: String = row.get(0).map_err(storage_error)?; + let entity_id: String = row.get(1).map_err(storage_error)?; + let sequence_nr: i64 = row.get(2).map_err(storage_error)?; + let projection_hash: String = row.get(3).map_err(storage_error)?; + existing.insert((entity_type, entity_id), (sequence_nr, projection_hash)); + } + drop(existing_rows); + } + + struct PreparedProjection { + entity_type: String, + entity_id: String, + status: String, + fields_json: String, + state_json: String, + sequence_nr: i64, + projection_hash: String, + source_backed: bool, + } + + let mut catalog_upserts = Vec::new(); + let mut changed_indexes = Vec::new(); + for projection in projections { + let new_projection_hash = projection_hash(&projection.status, &projection.fields); + let fields_json = serde_json::to_string(&projection.fields) + .map_err(|error| PersistenceError::Serialization(error.to_string()))?; + let state_json = serde_json::to_string(&projection.state) + .map_err(|error| PersistenceError::Serialization(error.to_string()))?; + let sequence_nr = i64::try_from(projection.sequence_nr).unwrap_or(i64::MAX); + let key = (projection.entity_type.clone(), projection.entity_id.clone()); + let existing_row = existing.get(&key); + let journal_sequence = journal_sequences.get(&key).copied().unwrap_or(0); + let source_backed = journal_sequence > 0 || snapshot_entities.contains(&key); + let incoming_is_stale = if journal_sequence > 0 { + sequence_nr != journal_sequence + } else { + existing_row.is_some_and(|(existing_sequence, _)| *existing_sequence > sequence_nr) + }; + if incoming_is_stale { + continue; + } + if existing_row.is_none_or(|(_, existing_hash)| existing_hash != &new_projection_hash) { + changed_indexes.push(( + projection.entity_type.clone(), + projection.entity_id.clone(), + projection.status.clone(), + indexed_projection_fields(&projection.status, &projection.indexed_fields), + false, + )); + } + catalog_upserts.push(PreparedProjection { + entity_type: projection.entity_type.clone(), + entity_id: projection.entity_id.clone(), + status: projection.status.clone(), + fields_json, + state_json, + sequence_nr, + projection_hash: new_projection_hash, + source_backed, + }); + } + + if !catalog_upserts.is_empty() { + let mut upsert_sql = String::from( + "INSERT INTO entity_catalog \ + (tenant, entity_type, entity_id, status, fields, state, updated_at, sequence_nr, projection_version, projection_hash) \ + VALUES ", + ); + let mut upsert_values = Vec::with_capacity(catalog_upserts.len() * 9); + for (index, projection) in catalog_upserts.iter().enumerate() { + if index > 0 { + upsert_sql.push_str(", "); + } + upsert_sql.push_str("(?, ?, ?, ?, ?, ?, ?, ?, 2, ?)"); + upsert_values.push(Value::from(tenant.to_string())); + upsert_values.push(Value::from(projection.entity_type.clone())); + upsert_values.push(Value::from(projection.entity_id.clone())); + upsert_values.push(Value::from(projection.status.clone())); + upsert_values.push(Value::from(projection.fields_json.clone())); + upsert_values.push(Value::from(projection.state_json.clone())); + upsert_values.push(Value::from(updated_at.clone())); + upsert_values.push(Value::from(projection.sequence_nr)); + upsert_values.push(Value::from(projection.projection_hash.clone())); + } + upsert_sql.push_str( + " ON CONFLICT(tenant, entity_type, entity_id) DO UPDATE SET \ + status = excluded.status, \ + fields = excluded.fields, \ + state = excluded.state, \ + updated_at = excluded.updated_at, \ + sequence_nr = excluded.sequence_nr, \ + projection_version = excluded.projection_version, \ + projection_hash = excluded.projection_hash", + ); + tx.execute(&upsert_sql, params_from_iter(upsert_values)) + .await + .map_err(storage_error)?; + } + + if !changed_indexes.is_empty() { + let delete_targets = changed_indexes.iter().collect::>(); + if !delete_targets.is_empty() { + let mut delete_sql = + String::from("DELETE FROM entity_field_index WHERE tenant = ? AND ("); + let mut delete_values = Vec::with_capacity(1 + delete_targets.len() * 2); + delete_values.push(Value::from(tenant.to_string())); + for (index, (entity_type, entity_id, _, _, _)) in delete_targets.iter().enumerate() + { + if index > 0 { + delete_sql.push_str(" OR "); + } + delete_sql.push_str("(entity_type = ? AND entity_id = ?)"); + delete_values.push(Value::from(entity_type.clone())); + delete_values.push(Value::from(entity_id.clone())); + } + delete_sql.push(')'); + tx.execute(&delete_sql, params_from_iter(delete_values)) + .await + .map_err(storage_error)?; + } + + let indexed_field_count = changed_indexes + .iter() + .map(|(_, _, _, fields, _)| fields.len()) + .sum::(); + if indexed_field_count > 0 { + let mut insert_sql = String::from( + "INSERT INTO entity_field_index \ + (tenant, entity_type, entity_id, field_name, field_value, status) VALUES ", + ); + let mut insert_values = Vec::with_capacity(indexed_field_count * 6); + let mut field_index = 0usize; + for (entity_type, entity_id, status, indexed_fields, _) in &changed_indexes { + for (field_name, field_value) in indexed_fields { + if field_index > 0 { + insert_sql.push_str(", "); + } + insert_sql.push_str("(?, ?, ?, ?, ?, ?)"); + insert_values.push(Value::from(tenant.to_string())); + insert_values.push(Value::from(entity_type.clone())); + insert_values.push(Value::from(entity_id.clone())); + insert_values.push(Value::from(field_name.clone())); + insert_values.push( + field_value + .as_ref() + .map(|value| Value::from(value.clone())) + .unwrap_or(Value::Null), + ); + insert_values.push(Value::from(status.clone())); + field_index += 1; + } + } + + tx.execute(&insert_sql, params_from_iter(insert_values)) + .await + .map_err(storage_error)?; + } + } + + for projection in &catalog_upserts { + if projection.source_backed { + mark_query_projection_dirty( + &tx, + tenant, + &projection.entity_type, + &projection.entity_id, + ) + .await?; + } else { + clear_query_projection_dirty( + &tx, + tenant, + &projection.entity_type, + &projection.entity_id, + ) + .await?; + } + } + + tx.commit().await.map_err(storage_error)?; + Ok(()) + } +} diff --git a/crates/temper-store-turso/src/store/field_index/projection_repair.rs b/crates/temper-store-turso/src/store/field_index/projection_repair.rs new file mode 100644 index 000000000..7c5db8fe1 --- /dev/null +++ b/crates/temper-store-turso/src/store/field_index/projection_repair.rs @@ -0,0 +1,277 @@ +//! Conditional query-projection removal and dirty repair. + +use super::*; + +impl TursoEventStore { + /// Remove the durable query-plane projection for a single entity. + #[instrument(skip_all, fields( + otel.name = "turso.remove_query_projection", + tenant, entity_type, entity_id, + ))] + pub async fn remove_query_projection( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + ) -> Result<(), PersistenceError> { + self.remove_query_projection_inner(tenant, entity_type, entity_id, None) + .await + .map(|_| ()) + } + + /// Remove a projection only while its exact journal/snapshot source is current. + pub async fn remove_query_projection_if_source( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + source: ProjectionSourceFence<'_>, + ) -> Result { + self.remove_query_projection_inner(tenant, entity_type, entity_id, Some(source)) + .await + } + + /// Clear a dirty marker only while its exact journal/snapshot source is current. + pub async fn clear_query_projection_dirty_if_source( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + source: ProjectionSourceFence<'_>, + ) -> Result { + let _write_permit = self + .acquire_write_permit( + "turso.clear_query_projection_dirty_if_source", + WritePriority::Low, + ) + .await?; + let conn = self.configured_connection().await?; + let tx = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .map_err(storage_error)?; + let journal_sequence = { + let mut rows = tx + .query( + "SELECT COALESCE(MAX(sequence_nr), 0) FROM events \ + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", + params![tenant, entity_type, entity_id], + ) + .await + .map_err(storage_error)?; + rows.next() + .await + .map_err(storage_error)? + .map(|row| row.get::(0).map_err(storage_error)) + .transpose()? + .unwrap_or(0) + }; + let current_snapshot = { + let mut rows = tx + .query( + "SELECT sequence_nr, snapshot FROM snapshots \ + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", + params![tenant, entity_type, entity_id], + ) + .await + .map_err(storage_error)?; + rows.next() + .await + .map_err(storage_error)? + .map(|row| { + Ok::<_, PersistenceError>(( + row.get::(0).map_err(storage_error)?, + row.get::>(1).map_err(storage_error)?, + )) + }) + .transpose()? + }; + let journal_matches = + u64::try_from(journal_sequence).ok() == Some(source.expected_journal_sequence); + let snapshot_matches = match (source.expected_snapshot, current_snapshot.as_ref()) { + (None, None) => true, + (Some(expected), Some((sequence_nr, snapshot))) => { + u64::try_from(*sequence_nr).ok() == Some(expected.sequence_nr) + && snapshot.as_slice() == expected.state + } + _ => false, + }; + if !journal_matches || !snapshot_matches { + tx.commit().await.map_err(storage_error)?; + return Ok(false); + } + clear_query_projection_dirty(&tx, tenant, entity_type, entity_id).await?; + tx.commit().await.map_err(storage_error)?; + Ok(true) + } + + async fn remove_query_projection_inner( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + source: Option>, + ) -> Result { + let source_fenced = source.is_some(); + let _write_permit = self + .acquire_write_permit("turso.remove_query_projection", WritePriority::Low) + .await?; + let conn = self.configured_connection().await?; + let tx = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .map_err(storage_error)?; + let journal_sequence = { + let mut journal_rows = tx + .query( + "SELECT COALESCE(MAX(sequence_nr), 0) FROM events \ + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", + params![tenant, entity_type, entity_id], + ) + .await + .map_err(storage_error)?; + journal_rows + .next() + .await + .map_err(storage_error)? + .map(|row| row.get::(0).map_err(storage_error)) + .transpose()? + .unwrap_or(0) + }; + let current_snapshot = { + let mut snapshot_rows = tx + .query( + "SELECT sequence_nr, snapshot FROM snapshots \ + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", + params![tenant, entity_type, entity_id], + ) + .await + .map_err(storage_error)?; + snapshot_rows + .next() + .await + .map_err(storage_error)? + .map(|row| { + Ok::<_, PersistenceError>(( + row.get::(0).map_err(storage_error)?, + row.get::>(1).map_err(storage_error)?, + )) + }) + .transpose()? + }; + let source_backed = journal_sequence > 0 || current_snapshot.is_some(); + if let Some(source) = source { + let journal_matches = + u64::try_from(journal_sequence).ok() == Some(source.expected_journal_sequence); + let snapshot_matches = match (source.expected_snapshot, current_snapshot.as_ref()) { + (None, None) => true, + (Some(expected), Some((sequence_nr, snapshot))) => { + u64::try_from(*sequence_nr).ok() == Some(expected.sequence_nr) + && snapshot.as_slice() == expected.state + } + _ => false, + }; + if !journal_matches || !snapshot_matches { + tx.commit().await.map_err(storage_error)?; + return Ok(false); + } + } + tx.execute( + "DELETE FROM entity_catalog WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", + params![tenant, entity_type, entity_id], + ) + .await + .map_err(storage_error)?; + tx.execute( + "DELETE FROM entity_field_index WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", + params![tenant, entity_type, entity_id], + ) + .await + .map_err(storage_error)?; + if source_fenced { + clear_query_projection_dirty(&tx, tenant, entity_type, entity_id).await?; + } else if source_backed { + mark_query_projection_dirty(&tx, tenant, entity_type, entity_id).await?; + } else { + clear_query_projection_dirty(&tx, tenant, entity_type, entity_id).await?; + } + tx.commit().await.map_err(storage_error)?; + Ok(true) + } + + /// Remove only an exact attempted projection row during unstable-source cleanup. + #[expect( + clippy::too_many_arguments, + reason = "exact projection cleanup boundary" + )] + pub async fn remove_query_projection_if_exact( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + status: &str, + fields: &serde_json::Value, + state: &serde_json::Value, + sequence_nr: u64, + ) -> Result { + let _write_permit = self + .acquire_write_permit("turso.remove_query_projection_if_exact", WritePriority::Low) + .await?; + let status = canonical_projection_status(status, state); + let fields_json = serde_json::to_string(fields) + .map_err(|error| PersistenceError::Serialization(error.to_string()))?; + let state_json = serde_json::to_string(state) + .map_err(|error| PersistenceError::Serialization(error.to_string()))?; + let sequence_nr = i64::try_from(sequence_nr) + .map_err(|_| PersistenceError::Storage("projection sequence exceeds i64".into()))?; + let conn = self.configured_connection().await?; + let tx = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .map_err(storage_error)?; + let removed = tx + .execute( + "DELETE FROM entity_catalog \ + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3 \ + AND status = ?4 AND fields = ?5 AND state = ?6 AND sequence_nr = ?7", + params![ + tenant, + entity_type, + entity_id, + status, + fields_json.as_str(), + state_json.as_str(), + sequence_nr, + ], + ) + .await + .map_err(storage_error)?; + if removed > 0 { + tx.execute( + "DELETE FROM entity_field_index \ + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", + params![tenant, entity_type, entity_id], + ) + .await + .map_err(storage_error)?; + mark_query_projection_dirty(&tx, tenant, entity_type, entity_id).await?; + } + tx.commit().await.map_err(storage_error)?; + Ok(removed > 0) + } + + /// Backwards-compatible alias for the old name. + #[instrument(skip_all, fields( + otel.name = "turso.remove_field_index", + tenant, entity_type, entity_id, + ))] + pub async fn remove_field_index( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + ) -> Result<(), PersistenceError> { + self.remove_query_projection(tenant, entity_type, entity_id) + .await + } +} diff --git a/crates/temper-store-turso/src/store/field_index/projection_write.rs b/crates/temper-store-turso/src/store/field_index/projection_write.rs new file mode 100644 index 000000000..f25a47a7a --- /dev/null +++ b/crates/temper-store-turso/src/store/field_index/projection_write.rs @@ -0,0 +1,396 @@ +//! Source-fenced query-projection writes. + +use super::*; + +impl TursoEventStore { + /// Upsert the durable query-plane projection for a single entity. + /// + /// Maintains both `entity_catalog` and the EAV `entity_field_index`. + #[instrument(skip_all, fields( + otel.name = "turso.upsert_query_projection", + tenant, entity_type, entity_id, + ))] + pub async fn upsert_query_projection( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + status: &str, + fields: &serde_json::Value, + sequence_nr: u64, + ) -> Result<(), PersistenceError> { + let state = serde_json::json!({ + "entity_type": entity_type, + "entity_id": entity_id, + "status": status, + "item_count": 0, + "counters": {}, + "booleans": {}, + "lists": {}, + "fields": fields, + "events": [], + "total_event_count": sequence_nr, + "sequence_nr": sequence_nr + }); + self.upsert_query_projection_with_state( + tenant, + entity_type, + entity_id, + status, + fields, + &state, + sequence_nr, + ) + .await + } + + #[expect(clippy::too_many_arguments, reason = "projection upsert boundary")] + pub async fn upsert_query_projection_with_state( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + status: &str, + fields: &serde_json::Value, + state: &serde_json::Value, + sequence_nr: u64, + ) -> Result<(), PersistenceError> { + self.upsert_query_projection_with_state_inner( + tenant, + entity_type, + entity_id, + status, + fields, + state, + sequence_nr, + None, + ) + .await + .map(|_| ()) + } + + /// Upsert a projection only while its exact journal/snapshot source is current. + #[expect( + clippy::too_many_arguments, + reason = "source-fenced projection boundary" + )] + pub async fn upsert_query_projection_with_state_if_source( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + status: &str, + fields: &serde_json::Value, + state: &serde_json::Value, + sequence_nr: u64, + source: ProjectionSourceFence<'_>, + ) -> Result { + self.upsert_query_projection_with_state_inner( + tenant, + entity_type, + entity_id, + status, + fields, + state, + sequence_nr, + Some(source), + ) + .await + } + + #[expect(clippy::too_many_arguments, reason = "projection upsert boundary")] + async fn upsert_query_projection_with_state_inner( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + status: &str, + fields: &serde_json::Value, + state: &serde_json::Value, + sequence_nr: u64, + source: Option>, + ) -> Result { + let source_fenced = source.is_some(); + let status = canonical_projection_status(status, state); + let _write_permit = self + .acquire_write_permit("turso.upsert_query_projection", WritePriority::Low) + .await?; + let conn = self.configured_connection().await?; + let new_projection_hash = projection_hash(status, fields); + let fields_json = serde_json::to_string(fields) + .map_err(|error| PersistenceError::Serialization(error.to_string()))?; + let state_json = serde_json::to_string(state) + .map_err(|error| PersistenceError::Serialization(error.to_string()))?; + let updated_at = sim_now().to_rfc3339(); + let sequence_nr = i64::try_from(sequence_nr).unwrap_or(i64::MAX); + + let tx = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .map_err(storage_error)?; + + let journal_sequence = { + let mut journal_rows = tx + .query( + "SELECT COALESCE(MAX(sequence_nr), 0) FROM events \ + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", + params![tenant, entity_type, entity_id], + ) + .await + .map_err(storage_error)?; + journal_rows + .next() + .await + .map_err(storage_error)? + .map(|row| row.get::(0).map_err(storage_error)) + .transpose()? + .unwrap_or(0) + }; + let source_backed = if journal_sequence > 0 { + true + } else { + let mut rows = tx + .query( + "SELECT EXISTS(SELECT 1 FROM snapshots \ + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3)", + params![tenant, entity_type, entity_id], + ) + .await + .map_err(storage_error)?; + rows.next() + .await + .map_err(storage_error)? + .map(|row| { + row.get::(0) + .map(|exists| exists != 0) + .map_err(storage_error) + }) + .transpose()? + .unwrap_or(false) + }; + + if let Some(source) = source { + let journal_matches = + u64::try_from(journal_sequence).ok() == Some(source.expected_journal_sequence); + let current_snapshot = { + let mut snapshot_rows = tx + .query( + "SELECT sequence_nr, snapshot FROM snapshots \ + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", + params![tenant, entity_type, entity_id], + ) + .await + .map_err(storage_error)?; + snapshot_rows + .next() + .await + .map_err(storage_error)? + .map(|row| { + Ok::<_, PersistenceError>(( + row.get::(0).map_err(storage_error)?, + row.get::>(1).map_err(storage_error)?, + )) + }) + .transpose()? + }; + let snapshot_matches = match (source.expected_snapshot, current_snapshot.as_ref()) { + (None, None) => true, + (Some(expected), Some((sequence_nr, snapshot))) => { + u64::try_from(*sequence_nr).ok() == Some(expected.sequence_nr) + && snapshot.as_slice() == expected.state + } + _ => false, + }; + let source_sequence = if source.expected_journal_sequence > 0 { + source.expected_journal_sequence + } else { + source + .expected_snapshot + .map(|snapshot| snapshot.sequence_nr) + .unwrap_or(0) + }; + if !journal_matches + || !snapshot_matches + || u64::try_from(sequence_nr).ok() != Some(source_sequence) + { + tx.commit().await.map_err(storage_error)?; + return Ok(false); + } + } + + let existing_sequence = { + let mut existing_rows = tx + .query( + "SELECT sequence_nr FROM entity_catalog \ + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", + params![tenant, entity_type, entity_id], + ) + .await + .map_err(storage_error)?; + if let Some(row) = existing_rows.next().await.map_err(storage_error)? { + Some(row.get::(0).map_err(storage_error)?) + } else { + None + } + }; + let incoming_is_stale = if source_fenced { + false + } else if journal_sequence > 0 { + sequence_nr != journal_sequence + } else { + existing_sequence.is_some_and(|existing| existing > sequence_nr) + }; + if incoming_is_stale { + tx.commit().await.map_err(storage_error)?; + return Ok(false); + } + + let unchanged_rows = tx + .execute( + "UPDATE entity_catalog \ + SET status = ?4, updated_at = ?5, sequence_nr = ?6, projection_version = 2, fields = ?8, state = ?9 \ + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3 AND projection_hash = ?7", + params![ + tenant, + entity_type, + entity_id, + status, + updated_at.as_str(), + sequence_nr, + new_projection_hash.as_str(), + fields_json.as_str(), + state_json.as_str(), + ], + ) + .await + .map_err(storage_error)?; + if unchanged_rows > 0 { + if source_fenced { + clear_query_projection_dirty(&tx, tenant, entity_type, entity_id).await?; + } else if source_backed { + mark_query_projection_dirty(&tx, tenant, entity_type, entity_id).await?; + } else { + clear_query_projection_dirty(&tx, tenant, entity_type, entity_id).await?; + } + tx.commit().await.map_err(storage_error)?; + return Ok(true); + } + + tx.execute( + "INSERT INTO entity_catalog \ + (tenant, entity_type, entity_id, status, fields, state, updated_at, sequence_nr, projection_version, projection_hash) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, 2, ?9) \ + ON CONFLICT(tenant, entity_type, entity_id) DO UPDATE SET \ + status = excluded.status, \ + fields = excluded.fields, \ + state = excluded.state, \ + updated_at = excluded.updated_at, \ + sequence_nr = excluded.sequence_nr, \ + projection_version = excluded.projection_version, \ + projection_hash = excluded.projection_hash", + params![ + tenant, + entity_type, + entity_id, + status, + fields_json.as_str(), + state_json.as_str(), + updated_at.as_str(), + sequence_nr, + new_projection_hash.as_str(), + ], + ) + .await + .map_err(storage_error)?; + + // Delete existing rows for this entity, then re-insert. + // This is simpler than tracking individual field changes and handles + // field removal correctly. + tx.execute( + "DELETE FROM entity_field_index WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", + params![tenant, entity_type, entity_id], + ) + .await + .map_err(storage_error)?; + + let indexed_fields = indexed_projection_fields(status, fields); + if !indexed_fields.is_empty() { + let mut sql = String::from( + "INSERT INTO entity_field_index \ + (tenant, entity_type, entity_id, field_name, field_value, status) VALUES ", + ); + let mut values = Vec::with_capacity(indexed_fields.len() * 6); + + for (index, (field_name, field_value)) in indexed_fields.iter().enumerate() { + if index > 0 { + sql.push_str(", "); + } + sql.push_str("(?, ?, ?, ?, ?, ?)"); + values.push(Value::from(tenant.to_string())); + values.push(Value::from(entity_type.to_string())); + values.push(Value::from(entity_id.to_string())); + values.push(Value::from(field_name.clone())); + values.push( + field_value + .as_ref() + .map(|value| Value::from(value.clone())) + .unwrap_or(Value::Null), + ); + values.push(Value::from(status.to_string())); + } + + tx.execute(&sql, params_from_iter(values)) + .await + .map_err(storage_error)?; + } + + if source_fenced { + clear_query_projection_dirty(&tx, tenant, entity_type, entity_id).await?; + } else if source_backed { + mark_query_projection_dirty(&tx, tenant, entity_type, entity_id).await?; + } else { + clear_query_projection_dirty(&tx, tenant, entity_type, entity_id).await?; + } + + tx.commit().await.map_err(storage_error)?; + Ok(true) + } + + /// Backwards-compatible alias for the old name. + #[instrument(skip_all, fields( + otel.name = "turso.upsert_field_index", + tenant, entity_type, entity_id, + ))] + pub async fn upsert_field_index( + &self, + tenant: &str, + entity_type: &str, + entity_id: &str, + status: &str, + fields: &serde_json::Value, + ) -> Result<(), PersistenceError> { + let state = serde_json::json!({ + "entity_type": entity_type, + "entity_id": entity_id, + "status": status, + "item_count": 0, + "counters": {}, + "booleans": {}, + "lists": {}, + "fields": fields, + "events": [], + "total_event_count": 0, + "sequence_nr": 0 + }); + self.upsert_query_projection_with_state( + tenant, + entity_type, + entity_id, + status, + fields, + &state, + 0, + ) + .await + } +} diff --git a/crates/temper-store-turso/src/store/mod.rs b/crates/temper-store-turso/src/store/mod.rs index fba0d839d..b6fdd5b3e 100644 --- a/crates/temper-store-turso/src/store/mod.rs +++ b/crates/temper-store-turso/src/store/mod.rs @@ -9,7 +9,7 @@ //! - [`constraints`]: Tenant-level cross-entity constraints //! - [`event_store`]: [`EventStore`] trait implementation -use libsql::{Builder, Database}; +use libsql::{Builder, Database, TransactionBehavior}; use std::sync::Arc; use std::sync::atomic::AtomicUsize; use temper_runtime::persistence::{PersistenceError, storage_error}; @@ -134,6 +134,9 @@ impl TursoEventStore { conn.execute(schema::CREATE_EVENTS_ENTITY_INDEX, ()) .await .map_err(storage_error)?; + conn.execute(schema::CREATE_PERSISTENCE_BATCH_IDEMPOTENCY_TABLE, ()) + .await + .map_err(storage_error)?; conn.execute(schema::CREATE_EVENT_SEGMENTS_TABLE, ()) .await .map_err(storage_error)?; @@ -381,6 +384,41 @@ impl TursoEventStore { conn.execute(schema::CREATE_ENTITY_FIELD_INDEX_STATUS, ()) .await .map_err(storage_error)?; + // Publish the ledger and its upgrade seed in one transaction. If the + // process stops anywhere before commit, SQLite/libSQL rolls back the + // table creation too, so the next startup cannot mistake an unseeded + // table for a completed migration. + let projection_dirty_tx = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .map_err(storage_error)?; + let query_projection_dirty_was_missing = { + let mut rows = projection_dirty_tx + .query( + "SELECT 1 FROM sqlite_schema \ + WHERE type = 'table' AND name = 'query_projection_dirty' \ + LIMIT 1", + (), + ) + .await + .map_err(storage_error)?; + rows.next().await.map_err(storage_error)?.is_none() + }; + projection_dirty_tx + .execute(schema::CREATE_QUERY_PROJECTION_DIRTY_TABLE, ()) + .await + .map_err(storage_error)?; + projection_dirty_tx + .execute(schema::CREATE_QUERY_PROJECTION_DIRTY_TYPE_INDEX, ()) + .await + .map_err(storage_error)?; + if query_projection_dirty_was_missing { + projection_dirty_tx + .execute(schema::SEED_QUERY_PROJECTION_DIRTY, ()) + .await + .map_err(storage_error)?; + } + projection_dirty_tx.commit().await.map_err(storage_error)?; // Entity key index (ADR-0153) — declared composite-key -> entity_id, the // negative-existence access path co-committed with the journal append. @@ -430,7 +468,7 @@ impl TursoEventStore { // Row / result types // --------------------------------------------------------------------------- -pub use policy::PolicyRow; +pub use policy::{PolicyGenerationWrite, PolicyRow}; /// Durable denial-pattern row used to rebuild policy suggestions. #[derive(Debug, Clone, serde::Serialize)] diff --git a/crates/temper-store-turso/src/store/policy.rs b/crates/temper-store-turso/src/store/policy.rs index 09dba4674..1a9cd56a8 100644 --- a/crates/temper-store-turso/src/store/policy.rs +++ b/crates/temper-store-turso/src/store/policy.rs @@ -5,15 +5,15 @@ //! per tenant), this table supports multiple entries per tenant, each with its own //! `policy_id`, content hash, enabled flag, and audit fields. -use libsql::params; +use libsql::{TransactionBehavior, params}; use sha2::{Digest, Sha256}; use temper_runtime::persistence::{PersistenceError, storage_error}; use tracing::instrument; -use super::{PolicyDenialPatternRow, TursoEventStore}; +use super::{TursoEventStore, write_gate::WritePriority}; use crate::metrics::TursoQueryTimer; -const DISTINCT_RESOURCE_IDS_BUDGET: usize = 100; +mod denial_patterns; /// A row from the `policies` table. #[derive(Debug, Clone)] @@ -35,7 +35,98 @@ pub struct PolicyRow { pub enabled: bool, } +/// One entry in an atomically published tenant policy generation. +#[derive(Debug, Clone)] +pub struct PolicyGenerationWrite<'a> { + /// Stable row identifier within the tenant. + pub policy_id: &'a str, + /// Raw Cedar source for the row. + pub cedar_text: &'a str, + /// Whether the row participates in the live generation. + pub enabled: bool, + /// Identity responsible for the row mutation. + pub created_by: &'a str, +} + impl TursoEventStore { + /// Load one tenant's compatibility policy projection. + pub async fn load_tenant_policy( + &self, + tenant: &str, + ) -> Result, PersistenceError> { + let conn = self.configured_connection().await?; + let mut rows = conn + .query( + "SELECT policy_text FROM tenant_policies WHERE tenant = ?1", + params![tenant], + ) + .await + .map_err(storage_error)?; + rows.next() + .await + .map_err(storage_error)? + .map(|row| row.get::(0).map_err(storage_error)) + .transpose() + } + + /// Atomically replace both representations of one tenant policy generation. + pub async fn replace_policy_generation( + &self, + tenant: &str, + entries: &[PolicyGenerationWrite<'_>], + compatibility_text: &str, + ) -> Result<(), PersistenceError> { + let mut policy_ids = std::collections::BTreeSet::new(); + for entry in entries { + if entry.policy_id.is_empty() || !policy_ids.insert(entry.policy_id) { + return Err(PersistenceError::Storage(format!( + "policy generation for tenant '{tenant}' contains an empty or duplicate policy id" + ))); + } + } + + let conn = self.configured_connection().await?; + let _write_permit = self + .acquire_write_permit("turso.replace_policy_generation", WritePriority::High) + .await?; + let tx = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .map_err(storage_error)?; + tx.execute("DELETE FROM policies WHERE tenant = ?1", params![tenant]) + .await + .map_err(storage_error)?; + for entry in entries { + let policy_hash = compute_policy_hash(entry.cedar_text); + let enabled = i32::from(entry.enabled); + tx.execute( + "INSERT INTO policies \ + (tenant, policy_id, cedar_text, policy_hash, created_at, created_by, enabled) \ + VALUES (?1, ?2, ?3, ?4, datetime('now'), ?5, ?6)", + params![ + tenant, + entry.policy_id, + entry.cedar_text, + policy_hash, + entry.created_by, + enabled + ], + ) + .await + .map_err(storage_error)?; + } + tx.execute( + "INSERT INTO tenant_policies (tenant, policy_text, updated_at) \ + VALUES (?1, ?2, datetime('now')) \ + ON CONFLICT(tenant) DO UPDATE SET \ + policy_text = excluded.policy_text, updated_at = datetime('now')", + params![tenant, compatibility_text], + ) + .await + .map_err(storage_error)?; + tx.commit().await.map_err(storage_error) + } + /// Persist a Cedar policy entry for a tenant. /// /// Computes a SHA-256 hash of `cedar_text` and compares it against any @@ -57,23 +148,30 @@ impl TursoEventStore { let policy_hash = compute_policy_hash(cedar_text); let conn = self.configured_connection().await?; - // Check existing hash to avoid redundant writes. - let existing_hash: Option = { + // A matching but disabled row is not a no-op: publication promises to + // make the candidate generation durable and live across restart. + let existing: Option<(String, bool)> = { let mut rows = conn .query( - "SELECT policy_hash FROM policies \ + "SELECT policy_hash, enabled FROM policies \ WHERE tenant = ?1 AND policy_id = ?2", params![tenant, policy_id], ) .await .map_err(storage_error)?; match rows.next().await.map_err(storage_error)? { - Some(row) => Some(row.get::(0).map_err(storage_error)?), + Some(row) => Some(( + row.get::(0).map_err(storage_error)?, + row.get::(1).map_err(storage_error)? != 0, + )), None => None, } }; - if existing_hash.as_deref() == Some(policy_hash.as_str()) { + if existing + .as_ref() + .is_some_and(|(hash, enabled)| hash == &policy_hash && *enabled) + { tracing::debug!( tenant, policy_id, @@ -90,7 +188,8 @@ impl TursoEventStore { cedar_text = excluded.cedar_text, \ policy_hash = excluded.policy_hash, \ created_by = excluded.created_by, \ - created_at = datetime('now')", + created_at = datetime('now'), \ + enabled = 1", params![ tenant, policy_id, @@ -181,140 +280,6 @@ impl TursoEventStore { Ok(out) } - /// Upsert a durable denial-pattern row for policy suggestion reconstruction. - #[instrument(skip_all, fields(tenant, action, resource_type, otel.name = "turso.upsert_policy_denial_pattern"))] - pub async fn upsert_policy_denial_pattern( - &self, - tenant: &str, - agent_type: Option<&str>, - action: &str, - resource_type: &str, - resource_id: &str, - timestamp: &str, - ) -> Result<(), PersistenceError> { - let _query_timer = TursoQueryTimer::start("turso.upsert_policy_denial_pattern"); - let conn = self.configured_connection().await?; - let agent_type_key = agent_type.unwrap_or(""); - - let existing = { - let mut rows = conn - .query( - "SELECT count, first_seen, last_seen, distinct_resource_ids_json \ - FROM policy_denial_patterns \ - WHERE tenant = ?1 AND agent_type = ?2 AND action = ?3 AND resource_type = ?4", - params![tenant, agent_type_key, action, resource_type], - ) - .await - .map_err(storage_error)?; - match rows.next().await.map_err(storage_error)? { - Some(row) => Some(( - row.get::(0).map_err(storage_error)?, - row.get::(1).map_err(storage_error)?, - row.get::(2).map_err(storage_error)?, - row.get::(3).map_err(storage_error)?, - )), - None => None, - } - }; - - let mut count = 1_i64; - let mut first_seen = timestamp.to_string(); - let mut last_seen = timestamp.to_string(); - let mut distinct_resource_ids = std::collections::BTreeSet::new(); - - if let Some((existing_count, existing_first_seen, existing_last_seen, ids_json)) = existing - { - count = existing_count + 1; - first_seen = existing_first_seen; - last_seen = if existing_last_seen.as_str() > timestamp { - existing_last_seen - } else { - timestamp.to_string() - }; - if let Ok(ids) = serde_json::from_str::>(&ids_json) { - distinct_resource_ids.extend(ids); - } - } - - distinct_resource_ids.insert(resource_id.to_string()); - while distinct_resource_ids.len() > DISTINCT_RESOURCE_IDS_BUDGET { - if let Some(oldest) = distinct_resource_ids.iter().next().cloned() { - distinct_resource_ids.remove(&oldest); - } else { - break; - } - } - - let ids_json = - serde_json::to_string(&distinct_resource_ids.into_iter().collect::>()) - .map_err(storage_error)?; - - conn.execute( - "INSERT INTO policy_denial_patterns \ - (tenant, agent_type, action, resource_type, count, first_seen, last_seen, distinct_resource_ids_json) \ - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8) \ - ON CONFLICT(tenant, agent_type, action, resource_type) DO UPDATE SET \ - count = excluded.count, \ - first_seen = excluded.first_seen, \ - last_seen = excluded.last_seen, \ - distinct_resource_ids_json = excluded.distinct_resource_ids_json", - params![ - tenant, - agent_type_key, - action, - resource_type, - count, - first_seen, - last_seen, - ids_json, - ], - ) - .await - .map_err(storage_error)?; - - Ok(()) - } - - /// Load durable denial patterns for one tenant, newest first. - #[instrument(skip_all, fields(tenant, otel.name = "turso.load_policy_denial_patterns"))] - pub async fn load_policy_denial_patterns( - &self, - tenant: &str, - ) -> Result, PersistenceError> { - let _query_timer = TursoQueryTimer::start("turso.load_policy_denial_patterns"); - let conn = self.configured_connection().await?; - let mut rows = conn - .query( - "SELECT tenant, agent_type, action, resource_type, count, first_seen, last_seen, distinct_resource_ids_json \ - FROM policy_denial_patterns \ - WHERE tenant = ?1 \ - ORDER BY last_seen DESC, count DESC", - params![tenant], - ) - .await - .map_err(storage_error)?; - - let mut out = Vec::new(); - while let Some(row) = rows.next().await.map_err(storage_error)? { - let agent_type_raw = row.get::(1).map_err(storage_error)?; - out.push(PolicyDenialPatternRow { - tenant: row.get::(0).map_err(storage_error)?, - agent_type: if agent_type_raw.is_empty() { - None - } else { - Some(agent_type_raw) - }, - action: row.get::(2).map_err(storage_error)?, - resource_type: row.get::(3).map_err(storage_error)?, - count: row.get::(4).map_err(storage_error)?, - first_seen: row.get::(5).map_err(storage_error)?, - last_seen: row.get::(6).map_err(storage_error)?, - distinct_resource_ids_json: row.get::(7).map_err(storage_error)?, - }); - } - Ok(out) - } - /// Toggle the `enabled` flag for a single Cedar policy entry. /// /// Returns `Ok(true)` if the row existed and was updated, `Ok(false)` if no @@ -367,6 +332,40 @@ impl TursoEventStore { Ok(affected > 0) } + /// Atomically replace the text and enabled state of one Cedar policy. + #[instrument(skip_all, fields(tenant, policy_id, otel.name = "turso.replace_policy"))] + pub async fn replace_policy( + &self, + tenant: &str, + policy_id: &str, + cedar_text: &str, + enabled: bool, + created_by: &str, + ) -> Result { + let _query_timer = TursoQueryTimer::start("turso.replace_policy"); + let policy_hash = compute_policy_hash(cedar_text); + let enabled_int: i32 = if enabled { 1 } else { 0 }; + let conn = self.configured_connection().await?; + let affected = conn + .execute( + "UPDATE policies \ + SET cedar_text = ?3, policy_hash = ?4, enabled = ?5, \ + created_by = ?6, created_at = datetime('now') \ + WHERE tenant = ?1 AND policy_id = ?2", + params![ + tenant, + policy_id, + cedar_text, + policy_hash, + enabled_int, + created_by + ], + ) + .await + .map_err(storage_error)?; + Ok(affected > 0) + } + /// Delete a single Cedar policy entry by `(tenant, policy_id)`. /// /// Silently succeeds if the row does not exist. diff --git a/crates/temper-store-turso/src/store/policy/denial_patterns.rs b/crates/temper-store-turso/src/store/policy/denial_patterns.rs new file mode 100644 index 000000000..897762bee --- /dev/null +++ b/crates/temper-store-turso/src/store/policy/denial_patterns.rs @@ -0,0 +1,146 @@ +//! Durable aggregation for denied Cedar authorization shapes. + +use libsql::params; +use temper_runtime::persistence::{PersistenceError, storage_error}; +use tracing::instrument; + +use super::super::{PolicyDenialPatternRow, TursoEventStore}; +use crate::metrics::TursoQueryTimer; + +const DISTINCT_RESOURCE_IDS_BUDGET: usize = 100; + +impl TursoEventStore { + /// Upsert a durable denial-pattern row for policy suggestion reconstruction. + #[instrument(skip_all, fields(tenant, action, resource_type, otel.name = "turso.upsert_policy_denial_pattern"))] + pub async fn upsert_policy_denial_pattern( + &self, + tenant: &str, + agent_type: Option<&str>, + action: &str, + resource_type: &str, + resource_id: &str, + timestamp: &str, + ) -> Result<(), PersistenceError> { + let _query_timer = TursoQueryTimer::start("turso.upsert_policy_denial_pattern"); + let conn = self.configured_connection().await?; + let agent_type_key = agent_type.unwrap_or(""); + + let existing = { + let mut rows = conn + .query( + "SELECT count, first_seen, last_seen, distinct_resource_ids_json \ + FROM policy_denial_patterns \ + WHERE tenant = ?1 AND agent_type = ?2 AND action = ?3 AND resource_type = ?4", + params![tenant, agent_type_key, action, resource_type], + ) + .await + .map_err(storage_error)?; + match rows.next().await.map_err(storage_error)? { + Some(row) => Some(( + row.get::(0).map_err(storage_error)?, + row.get::(1).map_err(storage_error)?, + row.get::(2).map_err(storage_error)?, + row.get::(3).map_err(storage_error)?, + )), + None => None, + } + }; + + let mut count = 1_i64; + let mut first_seen = timestamp.to_string(); + let mut last_seen = timestamp.to_string(); + let mut distinct_resource_ids = std::collections::BTreeSet::new(); + + if let Some((existing_count, existing_first_seen, existing_last_seen, ids_json)) = existing + { + count = existing_count + 1; + first_seen = existing_first_seen; + last_seen = if existing_last_seen.as_str() > timestamp { + existing_last_seen + } else { + timestamp.to_string() + }; + if let Ok(ids) = serde_json::from_str::>(&ids_json) { + distinct_resource_ids.extend(ids); + } + } + + distinct_resource_ids.insert(resource_id.to_string()); + while distinct_resource_ids.len() > DISTINCT_RESOURCE_IDS_BUDGET { + if let Some(oldest) = distinct_resource_ids.iter().next().cloned() { + distinct_resource_ids.remove(&oldest); + } else { + break; + } + } + + let ids_json = + serde_json::to_string(&distinct_resource_ids.into_iter().collect::>()) + .map_err(storage_error)?; + + conn.execute( + "INSERT INTO policy_denial_patterns \ + (tenant, agent_type, action, resource_type, count, first_seen, last_seen, distinct_resource_ids_json) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8) \ + ON CONFLICT(tenant, agent_type, action, resource_type) DO UPDATE SET \ + count = excluded.count, \ + first_seen = excluded.first_seen, \ + last_seen = excluded.last_seen, \ + distinct_resource_ids_json = excluded.distinct_resource_ids_json", + params![ + tenant, + agent_type_key, + action, + resource_type, + count, + first_seen, + last_seen, + ids_json, + ], + ) + .await + .map_err(storage_error)?; + + Ok(()) + } + + /// Load durable denial patterns for one tenant, newest first. + #[instrument(skip_all, fields(tenant, otel.name = "turso.load_policy_denial_patterns"))] + pub async fn load_policy_denial_patterns( + &self, + tenant: &str, + ) -> Result, PersistenceError> { + let _query_timer = TursoQueryTimer::start("turso.load_policy_denial_patterns"); + let conn = self.configured_connection().await?; + let mut rows = conn + .query( + "SELECT tenant, agent_type, action, resource_type, count, first_seen, last_seen, distinct_resource_ids_json \ + FROM policy_denial_patterns \ + WHERE tenant = ?1 \ + ORDER BY last_seen DESC, count DESC", + params![tenant], + ) + .await + .map_err(storage_error)?; + + let mut out = Vec::new(); + while let Some(row) = rows.next().await.map_err(storage_error)? { + let agent_type_raw = row.get::(1).map_err(storage_error)?; + out.push(PolicyDenialPatternRow { + tenant: row.get::(0).map_err(storage_error)?, + agent_type: if agent_type_raw.is_empty() { + None + } else { + Some(agent_type_raw) + }, + action: row.get::(2).map_err(storage_error)?, + resource_type: row.get::(3).map_err(storage_error)?, + count: row.get::(4).map_err(storage_error)?, + first_seen: row.get::(5).map_err(storage_error)?, + last_seen: row.get::(6).map_err(storage_error)?, + distinct_resource_ids_json: row.get::(7).map_err(storage_error)?, + }); + } + Ok(out) + } +} diff --git a/crates/temper-store-turso/src/store/specs.rs b/crates/temper-store-turso/src/store/specs.rs index 5edb0ab4e..f435eb59c 100644 --- a/crates/temper-store-turso/src/store/specs.rs +++ b/crates/temper-store-turso/src/store/specs.rs @@ -1,5 +1,7 @@ //! Spec persistence: upsert, verification updates, and startup loading. +mod publication; + use std::collections::BTreeMap; use libsql::{TransactionBehavior, params}; diff --git a/crates/temper-store-turso/src/store/specs/publication.rs b/crates/temper-store-turso/src/store/specs/publication.rs new file mode 100644 index 000000000..4b25033be --- /dev/null +++ b/crates/temper-store-turso/src/store/specs/publication.rs @@ -0,0 +1,256 @@ +//! Atomic durable publication for tenant spec generations. + +use sha2::{Digest, Sha256}; + +use super::*; + +impl TursoEventStore { + /// Atomically publish changed specs and delete replace-mode omissions. + #[expect( + clippy::too_many_arguments, + reason = "atomic publication boundary mirrors the platform-store contract" + )] + pub async fn publish_specs( + &self, + tenant: &str, + specs: &[(&str, &str, &str, &str)], + replace: bool, + constraints: Option>, + policy: Option<&str>, + os_app: Option<&TursoInstalledAppRow>, + wasm_modules: &[(&str, &[u8], &str, &str)], + policy_owner: Option<&str>, + policy_entries: &[(&str, &str, &str)], + ) -> Result, PersistenceError> { + let mut seen_types = std::collections::BTreeSet::new(); + for (entity_type, _, _, _) in specs { + if !seen_types.insert(*entity_type) { + return Err(PersistenceError::Storage(format!( + "duplicate entity type {entity_type} in spec publication for tenant {tenant}" + ))); + } + } + let mut seen_modules = std::collections::BTreeSet::new(); + for (module_name, _, _, _) in wasm_modules { + if !seen_modules.insert(*module_name) { + return Err(PersistenceError::Storage(format!( + "duplicate WASM module {module_name} in spec publication for tenant {tenant}" + ))); + } + } + let mut seen_policy_ids = std::collections::BTreeSet::new(); + for (policy_id, _, _) in policy_entries { + if !seen_policy_ids.insert(*policy_id) { + return Err(PersistenceError::Storage(format!( + "duplicate Cedar policy {policy_id} in spec publication for tenant {tenant}" + ))); + } + } + let conn = self.configured_connection().await?; + let _write_permit = self + .acquire_write_permit("turso.publish_specs", WritePriority::High) + .await?; + let tx = conn + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .map_err(storage_error)?; + let incoming_types = specs + .iter() + .map(|(entity_type, _, _, _)| *entity_type) + .collect::>(); + let removed_entity_types = if replace { + let mut rows = tx + .query( + "SELECT entity_type FROM specs WHERE tenant = ?1", + params![tenant], + ) + .await + .map_err(storage_error)?; + let mut removed = Vec::new(); + while let Some(row) = rows.next().await.map_err(storage_error)? { + let entity_type = row.get::(0).map_err(storage_error)?; + if !incoming_types.contains(entity_type.as_str()) { + removed.push(entity_type); + } + } + removed + } else { + Vec::new() + }; + for (entity_type, ioa_source, csdl_xml, content_hash) in specs { + tx.execute( + "INSERT INTO specs (tenant, entity_type, ioa_source, csdl_xml, content_hash, committed, version, verified, verification_status, updated_at) + VALUES (?1, ?2, ?3, ?4, ?5, 1, 1, 0, 'pending', datetime('now')) + ON CONFLICT (tenant, entity_type) DO UPDATE SET + ioa_source = CASE WHEN specs.content_hash IS NOT excluded.content_hash OR specs.csdl_xml IS NOT excluded.csdl_xml THEN excluded.ioa_source ELSE specs.ioa_source END, + csdl_xml = CASE WHEN specs.content_hash IS NOT excluded.content_hash OR specs.csdl_xml IS NOT excluded.csdl_xml THEN excluded.csdl_xml ELSE specs.csdl_xml END, + content_hash = CASE WHEN specs.content_hash IS NOT excluded.content_hash OR specs.csdl_xml IS NOT excluded.csdl_xml THEN excluded.content_hash ELSE specs.content_hash END, + committed = 1, + version = CASE WHEN specs.content_hash IS NOT excluded.content_hash OR specs.csdl_xml IS NOT excluded.csdl_xml THEN specs.version + 1 ELSE specs.version END, + verified = CASE WHEN specs.content_hash IS NOT excluded.content_hash OR specs.csdl_xml IS NOT excluded.csdl_xml THEN 0 ELSE specs.verified END, + verification_status = CASE WHEN specs.content_hash IS NOT excluded.content_hash OR specs.csdl_xml IS NOT excluded.csdl_xml THEN 'pending' ELSE specs.verification_status END, + levels_passed = CASE WHEN specs.content_hash IS NOT excluded.content_hash OR specs.csdl_xml IS NOT excluded.csdl_xml THEN NULL ELSE specs.levels_passed END, + levels_total = CASE WHEN specs.content_hash IS NOT excluded.content_hash OR specs.csdl_xml IS NOT excluded.csdl_xml THEN NULL ELSE specs.levels_total END, + verification_result = CASE WHEN specs.content_hash IS NOT excluded.content_hash OR specs.csdl_xml IS NOT excluded.csdl_xml THEN NULL ELSE specs.verification_result END, + updated_at = CASE WHEN specs.content_hash IS NOT excluded.content_hash OR specs.csdl_xml IS NOT excluded.csdl_xml OR specs.committed != 1 THEN datetime('now') ELSE specs.updated_at END", + params![tenant, *entity_type, *ioa_source, *csdl_xml, *content_hash], + ) + .await + .map_err(storage_error)?; + } + for entity_type in &removed_entity_types { + tx.execute( + "DELETE FROM specs WHERE tenant = ?1 AND entity_type = ?2", + params![tenant, entity_type.as_str()], + ) + .await + .map_err(storage_error)?; + } + match constraints { + None => {} + Some(Some(source)) => { + tx.execute( + "INSERT INTO tenant_constraints \ + (tenant, cross_invariants_toml, version, updated_at) \ + VALUES (?1, ?2, 1, datetime('now')) \ + ON CONFLICT (tenant) DO UPDATE SET \ + cross_invariants_toml = excluded.cross_invariants_toml, \ + version = tenant_constraints.version + 1, \ + updated_at = datetime('now')", + params![tenant, source], + ) + .await + .map_err(storage_error)?; + } + Some(None) => { + tx.execute( + "DELETE FROM tenant_constraints WHERE tenant = ?1", + params![tenant], + ) + .await + .map_err(storage_error)?; + } + } + if let Some(policy) = policy { + tx.execute( + "INSERT INTO tenant_policies (tenant, policy_text, updated_at) \ + VALUES (?1, ?2, datetime('now')) \ + ON CONFLICT(tenant) DO UPDATE SET \ + policy_text = excluded.policy_text, updated_at = datetime('now')", + params![tenant, policy], + ) + .await + .map_err(storage_error)?; + } + if let Some(owner) = policy_owner { + tx.execute( + "DELETE FROM policies WHERE tenant = ?1 AND created_by = ?2", + params![tenant, owner], + ) + .await + .map_err(storage_error)?; + for (policy_id, cedar_text, created_by) in policy_entries { + let mut digest = Sha256::new(); + digest.update(cedar_text.as_bytes()); + let policy_hash = format!("{:x}", digest.finalize()); + tx.execute( + "INSERT INTO policies \ + (tenant, policy_id, cedar_text, policy_hash, created_at, created_by, enabled) \ + VALUES (?1, ?2, ?3, ?4, datetime('now'), ?5, 1) \ + ON CONFLICT(tenant, policy_id) DO UPDATE SET \ + cedar_text = excluded.cedar_text, policy_hash = excluded.policy_hash, \ + created_by = excluded.created_by, created_at = datetime('now'), enabled = 1", + params![tenant, *policy_id, *cedar_text, policy_hash, *created_by], + ) + .await + .map_err(storage_error)?; + } + } + for (module_name, wasm_bytes, sha256_hash, source) in wasm_modules { + let replace_uploaded_wasm = *source == "bundled-replace-upload"; + let persisted_source = if replace_uploaded_wasm { + "bundled" + } else { + *source + }; + tx.execute( + "INSERT INTO wasm_modules \ + (tenant, module_name, wasm_bytes, sha256_hash, version, size_bytes, updated_at, source) \ + VALUES (?1, ?2, ?3, ?4, 1, ?5, datetime('now'), ?6) \ + ON CONFLICT(tenant, module_name) DO UPDATE SET \ + wasm_bytes = excluded.wasm_bytes, sha256_hash = excluded.sha256_hash, \ + version = wasm_modules.version + 1, size_bytes = excluded.size_bytes, \ + updated_at = datetime('now'), source = excluded.source \ + WHERE wasm_modules.sha256_hash IS NOT excluded.sha256_hash \ + AND (?7 OR excluded.source = 'upload' OR wasm_modules.source = 'bundled')", + params![ + tenant, + *module_name, + wasm_bytes.to_vec(), + *sha256_hash, + wasm_bytes.len() as i64, + persisted_source, + replace_uploaded_wasm, + ], + ) + .await + .map_err(storage_error)?; + } + if let Some(record) = os_app { + tx.execute( + "INSERT INTO tenant_installed_apps ( + tenant_id, app_name, source_kind, app_ref, version_hash, + pinned_version_hash, current_version_hash, follow_policy, closure_id, + registry_url, registry_tenant, app_version, bundle_digest, spec_digest, + policy_digest, wasm_digest, content_digest, seed_digest, + installed_at, last_reconciled_at, status + ) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, datetime('now'), datetime('now'), ?19) + ON CONFLICT(tenant_id, app_name) DO UPDATE SET + source_kind = excluded.source_kind, + app_ref = excluded.app_ref, + version_hash = excluded.version_hash, + pinned_version_hash = excluded.pinned_version_hash, + current_version_hash = excluded.current_version_hash, + follow_policy = excluded.follow_policy, + closure_id = excluded.closure_id, + registry_url = excluded.registry_url, + registry_tenant = excluded.registry_tenant, + app_version = excluded.app_version, + bundle_digest = excluded.bundle_digest, + spec_digest = excluded.spec_digest, + policy_digest = excluded.policy_digest, + wasm_digest = excluded.wasm_digest, + content_digest = excluded.content_digest, + seed_digest = excluded.seed_digest, + last_reconciled_at = datetime('now'), + status = excluded.status", + params![ + tenant, + record.app_name.as_str(), + record.source_kind.as_str(), + record.app_ref.as_str(), + record.version_hash.as_str(), + record.pinned_version_hash.as_str(), + record.current_version_hash.as_str(), + record.follow_policy.as_str(), + record.closure_id.as_str(), + record.registry_url.as_str(), + record.registry_tenant.as_str(), + record.app_version.as_str(), + record.bundle_digest.as_str(), + record.spec_digest.as_str(), + record.policy_digest.as_str(), + record.wasm_digest.as_str(), + record.content_digest.as_str(), + record.seed_digest.as_str(), + record.status.as_str(), + ], + ) + .await + .map_err(storage_error)?; + } + tx.commit().await.map_err(storage_error)?; + Ok(removed_entity_types) + } +} diff --git a/crates/temper-store-turso/src/store/tests/mod.rs b/crates/temper-store-turso/src/store/tests/mod.rs index f75aa4c01..28ed8bc25 100644 --- a/crates/temper-store-turso/src/store/tests/mod.rs +++ b/crates/temper-store-turso/src/store/tests/mod.rs @@ -1,14 +1,22 @@ //! Integration tests for the Turso event store. -use libsql::params; +use libsql::{TransactionBehavior, params}; use temper_runtime::persistence::{ - EntityVectorRow, EventMetadata, EventStore, PersistenceAppend, PersistenceEnvelope, - PersistenceError, + EntityVectorRow, EventMetadata, EventStore, IndexReconciliation, PersistenceAppend, + PersistenceEnvelope, PersistenceError, ProjectionSourceFence, SnapshotBackfillFence, + SnapshotSourceFence, }; use super::{PublishedArtifactUpsert, QueryProjectionUpsert, TursoEventStore}; use crate::TursoSpecVerificationUpdate; +const VECTOR_RECONCILIATION: IndexReconciliation = IndexReconciliation { + keys: false, + key_set_signature: None, + vectors: true, + snapshot_source: SnapshotSourceFence::Unchecked, +}; + fn test_envelope(event_type: &str, payload: serde_json::Value) -> PersistenceEnvelope { PersistenceEnvelope { sequence_nr: 0, @@ -39,6 +47,134 @@ async fn make_store(test_name: &str) -> TursoEventStore { .expect("create store") } +fn source_reconciliation(snapshot_source: SnapshotSourceFence) -> IndexReconciliation { + IndexReconciliation { + keys: false, + key_set_signature: None, + vectors: false, + snapshot_source, + } +} + +fn unchecked_batch_append( + persistence_id: &str, + expected_sequence: u64, + events: Vec, +) -> PersistenceAppend { + PersistenceAppend { + persistence_id: persistence_id.to_string(), + expected_sequence, + events, + key_rows: Vec::new(), + reconcile_keys: false, + key_set_signature: None, + snapshot_source: SnapshotSourceFence::Unchecked, + batch_idempotency: None, + } +} + +async fn segment_topology( + store: &TursoEventStore, + tenant: &str, + entity_type: &str, + entity_id: &str, +) -> Vec<(i64, i64, Option, Option, i64, i64)> { + let conn = store + .configured_connection() + .await + .expect("open topology connection"); + let mut rows = conn + .query( + "SELECT segment_index, start_sequence_nr, end_sequence_nr, + snapshot_sequence, event_count, sealed_at IS NOT NULL + FROM event_segments + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3 + ORDER BY segment_index", + params![tenant, entity_type, entity_id], + ) + .await + .expect("query segment topology"); + let mut out = Vec::new(); + while let Some(row) = rows.next().await.expect("read segment topology") { + out.push(( + row.get::(0).expect("segment index"), + row.get::(1).expect("segment start"), + row.get::>(2).expect("segment end"), + row.get::>(3).expect("segment snapshot"), + row.get::(4).expect("segment event count"), + row.get::(5).expect("segment sealed flag"), + )); + } + out +} + +async fn event_segment_assignments( + store: &TursoEventStore, + tenant: &str, + entity_type: &str, + entity_id: &str, +) -> Vec<(i64, i64)> { + let conn = store + .configured_connection() + .await + .expect("open event assignment connection"); + let mut rows = conn + .query( + "SELECT sequence_nr, segment_index + FROM events + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3 + ORDER BY sequence_nr", + params![tenant, entity_type, entity_id], + ) + .await + .expect("query event segment assignments"); + let mut out = Vec::new(); + while let Some(row) = rows.next().await.expect("read event segment assignment") { + out.push(( + row.get::(0).expect("event sequence"), + row.get::(1).expect("event segment"), + )); + } + out +} + +async fn seed_legacy_snapshot_ahead_segments( + store: &TursoEventStore, + tenant: &str, + entity_type: &str, + entity_id: &str, +) { + let conn = store + .configured_connection() + .await + .expect("open legacy segment connection"); + conn.execute( + "DELETE FROM event_segments + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3", + params![tenant, entity_type, entity_id], + ) + .await + .expect("clear current segment topology"); + conn.execute( + "INSERT INTO event_segments + (tenant, entity_type, entity_id, segment_index, start_sequence_nr, + end_sequence_nr, snapshot_sequence, event_count, sealed_at) + VALUES (?1, ?2, ?3, 0, 1, 5, 5, 5, datetime('now'))", + params![tenant, entity_type, entity_id], + ) + .await + .expect("seed legacy sealed segment"); + conn.execute( + "INSERT INTO event_segments + (tenant, entity_type, entity_id, segment_index, start_sequence_nr, + end_sequence_nr, snapshot_sequence, event_count, sealed_at) + VALUES (?1, ?2, ?3, 1, 6, NULL, NULL, 0, NULL)", + params![tenant, entity_type, entity_id], + ) + .await + .expect("seed legacy open segment"); +} + #[tokio::test] async fn append_and_read_events_roundtrip() { let store = make_store("append-read").await; @@ -85,7 +221,7 @@ async fn vector_index_write_behind_candidates_and_partitioning() { &[test_envelope("Create", serde_json::json!({}))], &[], &[row("embed", "m1", vec![0.0, 1.0])], - true, + VECTOR_RECONCILIATION, ) .await .unwrap(); @@ -96,7 +232,7 @@ async fn vector_index_write_behind_candidates_and_partitioning() { &[test_envelope("Create", serde_json::json!({}))], &[], &[row("embed", "m1", vec![1.0, 0.0])], - true, + VECTOR_RECONCILIATION, ) .await .unwrap(); @@ -108,7 +244,7 @@ async fn vector_index_write_behind_candidates_and_partitioning() { &[test_envelope("Create", serde_json::json!({}))], &[], &[row("embed", "m2", vec![1.0, 0.0])], - true, + VECTOR_RECONCILIATION, ) .await .unwrap(); @@ -172,7 +308,7 @@ async fn vector_index_reconcile_purges_on_delete_and_empty_rows() { &[test_envelope("Create", serde_json::json!({}))], &[], std::slice::from_ref(&row(vec![1.0, 0.0])), - true, + VECTOR_RECONCILIATION, ) .await .unwrap(); @@ -192,7 +328,7 @@ async fn vector_index_reconcile_purges_on_delete_and_empty_rows() { &[test_envelope("Delete", serde_json::json!({}))], &[], &[], - true, + VECTOR_RECONCILIATION, ) .await .unwrap(); @@ -282,6 +418,11 @@ async fn append_batch_zero_sequence_detects_existing_stream_by_unique_key() { "OrderUpdated", serde_json::json!({ "step": 2 }), )], + key_rows: Vec::new(), + reconcile_keys: false, + key_set_signature: None, + snapshot_source: SnapshotSourceFence::Unchecked, + batch_idempotency: None, }]) .await .unwrap_err(); @@ -299,6 +440,81 @@ async fn append_batch_zero_sequence_detects_existing_stream_by_unique_key() { assert_eq!(events[0].event_type, "OrderCreated"); } +#[tokio::test] +async fn append_batch_claim_is_atomic_content_bound_and_retryable() { + let store = make_store("append-batch-idempotency").await; + let persistence_id = "tenant-a:Repository:repo-idempotent"; + let mut append = unchecked_batch_append( + persistence_id, + 0, + vec![test_envelope( + "CompositeEvent", + serde_json::json!({"push": 1}), + )], + ); + append.batch_idempotency = Some(temper_runtime::persistence::PersistenceBatchIdempotency { + persistence_id: persistence_id.to_string(), + idempotency_key: "push-1".to_string(), + intent_hash: "intent-a".to_string(), + }); + + store + .append_batch(std::slice::from_ref(&append)) + .await + .expect("first batch"); + let retry = store + .append_batch(std::slice::from_ref(&append)) + .await + .expect("stale exact retry must be a no-op"); + assert!(retry[0].batch_already_applied); + assert_eq!(store.read_events(persistence_id, 0).await.unwrap().len(), 1); + + append + .batch_idempotency + .as_mut() + .expect("claim") + .intent_hash = "intent-b".to_string(); + let error = store + .append_batch(&[append]) + .await + .expect_err("same key with different work must fail"); + assert!(error.to_string().contains("different intent")); +} + +#[tokio::test] +async fn saving_an_unchanged_disabled_policy_durably_reenables_it() { + let store = make_store("policy-reenable").await; + let policy = "permit(principal, action, resource);"; + assert!( + store + .save_policy("tenant-a", "generated", policy, "test") + .await + .expect("create policy") + ); + assert!( + store + .toggle_policy_enabled("tenant-a", "generated", false) + .await + .expect("disable policy") + ); + + assert!( + store + .save_policy("tenant-a", "generated", policy, "test") + .await + .expect("republish identical policy") + ); + let rows = store + .load_policies_for_tenant("tenant-a") + .await + .expect("reload policies"); + assert_eq!(rows.len(), 1); + assert!( + rows[0].enabled, + "equal policy content must not suppress the durable enabled transition" + ); +} + #[tokio::test] async fn single_event_append_bypasses_process_write_gate() { let mut store = make_store("single-append-bypasses-gate").await; @@ -353,141 +569,691 @@ async fn snapshot_save_and_load_roundtrip() { } #[tokio::test] -async fn list_entity_ids_returns_distinct_pairs() { - let store = make_store("entity-list").await; +async fn delayed_snapshot_splits_tail_and_equal_replacement_preserves_segments() { + let store = make_store("delayed-snapshot-segments").await; + let tenant = "tenant-segments"; + let entity_type = "Order"; + let entity_id = "delayed-snapshot"; + let persistence_id = format!("{tenant}:{entity_type}:{entity_id}"); + let events = (1..=10) + .map(|sequence| test_envelope("Updated", serde_json::json!({"sequence": sequence}))) + .collect::>(); - let tenant_a = format!("tenant-a-{}", uuid::Uuid::new_v4()); - let tenant_b = format!("tenant-b-{}", uuid::Uuid::new_v4()); + store.append(&persistence_id, 0, &events).await.unwrap(); + store + .save_snapshot(&persistence_id, 5, b"snapshot-a") + .await + .unwrap(); - let order_1 = format!("{tenant_a}:Order:ord-1"); - let order_2 = format!("{tenant_a}:Order:ord-2"); - let task_1 = format!("{tenant_a}:Task:task-1"); - let other_tenant = format!("{tenant_b}:Order:ord-9"); + let expected_topology = vec![(0, 1, Some(5), Some(5), 5, 1), (1, 6, Some(10), None, 5, 0)]; + assert_eq!( + segment_topology(&store, tenant, entity_type, entity_id).await, + expected_topology + ); + let expected_assignments = (1..=10) + .map(|sequence| (sequence, if sequence <= 5 { 0 } else { 1 })) + .collect::>(); + assert_eq!( + event_segment_assignments(&store, tenant, entity_type, entity_id).await, + expected_assignments + ); - store - .append( - &order_1, - 0, - &[test_envelope( - "OrderCreated", - serde_json::json!({ "id": "ord-1" }), - )], + let audit_conn = store.configured_connection().await.unwrap(); + audit_conn + .execute_batch( + "CREATE TABLE snapshot_mutation_audit ( + target TEXT NOT NULL, + operation TEXT NOT NULL + ); + CREATE TRIGGER audit_snapshots_update AFTER UPDATE ON snapshots + BEGIN + INSERT INTO snapshot_mutation_audit VALUES ('snapshots', 'update'); + END; + CREATE TRIGGER audit_history_update AFTER UPDATE ON snapshot_history + BEGIN + INSERT INTO snapshot_mutation_audit VALUES ('snapshot_history', 'update'); + END; + CREATE TRIGGER audit_history_insert AFTER INSERT ON snapshot_history + BEGIN + INSERT INTO snapshot_mutation_audit VALUES ('snapshot_history', 'insert'); + END; + CREATE TRIGGER audit_events_update AFTER UPDATE ON events + BEGIN + INSERT INTO snapshot_mutation_audit VALUES ('events', 'update'); + END; + CREATE TRIGGER audit_segments_insert AFTER INSERT ON event_segments + BEGIN + INSERT INTO snapshot_mutation_audit VALUES ('event_segments', 'insert'); + END; + CREATE TRIGGER audit_segments_update AFTER UPDATE ON event_segments + BEGIN + INSERT INTO snapshot_mutation_audit VALUES ('event_segments', 'update'); + END; + CREATE TRIGGER audit_segments_delete AFTER DELETE ON event_segments + BEGIN + INSERT INTO snapshot_mutation_audit VALUES ('event_segments', 'delete'); + END;", ) .await .unwrap(); + + let topology_before_equal_replacement = + segment_topology(&store, tenant, entity_type, entity_id).await; + let assignments_before_equal_replacement = + event_segment_assignments(&store, tenant, entity_type, entity_id).await; store - .append( - &order_1, - 1, - &[test_envelope( - "OrderUpdated", - serde_json::json!({ "step": 2 }), - )], - ) + .save_snapshot(&persistence_id, 5, b"snapshot-b") .await .unwrap(); - store - .append( - &order_2, - 0, - &[test_envelope( - "OrderCreated", - serde_json::json!({ "id": "ord-2" }), - )], + assert_eq!( + segment_topology(&store, tenant, entity_type, entity_id).await, + topology_before_equal_replacement + ); + assert_eq!( + event_segment_assignments(&store, tenant, entity_type, entity_id).await, + assignments_before_equal_replacement + ); + assert_eq!( + store.load_snapshot(&persistence_id).await.unwrap(), + Some((5, b"snapshot-b".to_vec())) + ); + let mut audit_rows = audit_conn + .query( + "SELECT target, operation + FROM snapshot_mutation_audit + ORDER BY target, operation", + (), ) .await .unwrap(); + let mut audited_mutations = Vec::new(); + while let Some(row) = audit_rows.next().await.unwrap() { + audited_mutations.push((row.get::(0).unwrap(), row.get::(1).unwrap())); + } + assert_eq!( + audited_mutations, + vec![ + ("snapshot_history".to_string(), "update".to_string()), + ("snapshots".to_string(), "update".to_string()), + ], + "equal-sequence replacement may update snapshot bytes but not topology" + ); + audit_conn + .execute("DELETE FROM snapshot_mutation_audit", ()) + .await + .unwrap(); + + // Identical and older saves are complete no-ops: neither the current row, + // history, nor segment topology changes. store - .append( - &task_1, - 0, - &[test_envelope( - "TaskCreated", - serde_json::json!({ "id": "task-1" }), - )], - ) + .save_snapshot(&persistence_id, 5, b"snapshot-b") .await .unwrap(); store - .append( - &other_tenant, - 0, - &[test_envelope( - "OrderCreated", - serde_json::json!({ "id": "ord-9" }), - )], + .save_snapshot(&persistence_id, 4, b"snapshot-older") + .await + .unwrap(); + assert_eq!( + segment_topology(&store, tenant, entity_type, entity_id).await, + topology_before_equal_replacement + ); + assert_eq!( + store.load_snapshot(&persistence_id).await.unwrap(), + Some((5, b"snapshot-b".to_vec())) + ); + let mut audit_rows = audit_conn + .query("SELECT COUNT(*) FROM snapshot_mutation_audit", ()) + .await + .unwrap(); + let mutation_count = audit_rows + .next() + .await + .unwrap() + .expect("audit count") + .get::(0) + .unwrap(); + assert_eq!(mutation_count, 0, "older/identical saves must not write"); + + let conn = store.configured_connection().await.unwrap(); + let mut history_rows = conn + .query( + "SELECT sequence_nr, snapshot + FROM snapshot_history + WHERE tenant = ?1 AND entity_type = ?2 AND entity_id = ?3 + ORDER BY sequence_nr", + params![tenant, entity_type, entity_id], ) .await .unwrap(); + let history = history_rows.next().await.unwrap().expect("history row"); + assert_eq!(history.get::(0).unwrap(), 5); + assert_eq!(history.get::>(1).unwrap(), b"snapshot-b"); + assert!(history_rows.next().await.unwrap().is_none()); +} - let mut entities = store.list_entity_ids(&tenant_a).await.unwrap(); - entities.sort(); +#[tokio::test] +async fn append_batch_after_snapshot_uses_each_streams_open_segment() { + let store = make_store("append-batch-after-snapshot").await; + let tenant = "tenant-batch-segments"; + let entity_type = "Order"; + let entity_ids = ["batch-a", "batch-b"]; + let persistence_ids = entity_ids.map(|entity_id| format!("{tenant}:{entity_type}:{entity_id}")); + let initial_events = (1..=5) + .map(|sequence| test_envelope("Updated", serde_json::json!({"sequence": sequence}))) + .collect::>(); + + for persistence_id in &persistence_ids { + store + .append(persistence_id, 0, &initial_events) + .await + .unwrap(); + store + .save_snapshot(persistence_id, 3, b"snapshot-three") + .await + .unwrap(); + } + let results = store + .append_batch(&[ + unchecked_batch_append( + &persistence_ids[0], + 5, + vec![ + test_envelope("Updated", serde_json::json!({"sequence": 6})), + test_envelope("Updated", serde_json::json!({"sequence": 7})), + ], + ), + unchecked_batch_append( + &persistence_ids[1], + 5, + vec![ + test_envelope("Updated", serde_json::json!({"sequence": 6})), + test_envelope("Updated", serde_json::json!({"sequence": 7})), + ], + ), + ]) + .await + .unwrap(); assert_eq!( - entities, - vec![ - ("Order".to_string(), "ord-1".to_string()), - ("Order".to_string(), "ord-2".to_string()), - ("Task".to_string(), "task-1".to_string()), - ] + results + .iter() + .map(|result| result.sequence_nr) + .collect::>(), + vec![7, 7] ); + + let expected_topology = vec![(0, 1, Some(3), Some(3), 3, 1), (1, 4, Some(7), None, 4, 0)]; + let expected_assignments = (1..=7) + .map(|sequence| (sequence, if sequence <= 3 { 0 } else { 1 })) + .collect::>(); + for entity_id in entity_ids { + assert_eq!( + segment_topology(&store, tenant, entity_type, entity_id).await, + expected_topology + ); + assert_eq!( + event_segment_assignments(&store, tenant, entity_type, entity_id).await, + expected_assignments + ); + } } #[tokio::test] -async fn list_entity_ids_by_type_uses_entity_catalog() { - let store = make_store("entity-list-by-type-catalog").await; - let tenant = format!("tenant-{}", uuid::Uuid::new_v4()); +async fn snapshot_only_streams_start_their_first_journal_at_segment_zero() { + let store = make_store("snapshot-only-first-journal").await; + let tenant = "tenant-snapshot-only"; + let entity_type = "Order"; + let entity_ids = ["ordinary", "batch-a", "batch-b"]; + + for entity_id in entity_ids { + let persistence_id = format!("{tenant}:{entity_type}:{entity_id}"); + store + .save_snapshot(&persistence_id, 5, b"snapshot-five") + .await + .unwrap(); + assert!( + segment_topology(&store, tenant, entity_type, entity_id) + .await + .is_empty(), + "a snapshot without journal events must not manufacture segments" + ); + seed_legacy_snapshot_ahead_segments(&store, tenant, entity_type, entity_id).await; + } + let ordinary_id = format!("{tenant}:{entity_type}:ordinary"); store - .upsert_query_projection( - &tenant, - "AgentRoute", - "route-main", - "Ready", - &serde_json::json!({ "Name": "main" }), - 3, + .append( + &ordinary_id, + 0, + &[test_envelope("Created", serde_json::json!({"sequence": 1}))], ) .await - .expect("upsert AgentRoute projection"); + .unwrap(); + + let batch_a_id = format!("{tenant}:{entity_type}:batch-a"); + let batch_b_id = format!("{tenant}:{entity_type}:batch-b"); + store + .append_batch(&[ + unchecked_batch_append( + &batch_a_id, + 0, + vec![test_envelope("Created", serde_json::json!({"sequence": 1}))], + ), + unchecked_batch_append( + &batch_b_id, + 0, + vec![test_envelope("Created", serde_json::json!({"sequence": 1}))], + ), + ]) + .await + .unwrap(); + + let expected_topology = vec![(0, 1, Some(1), None, 1, 0)]; + let expected_assignments = vec![(1, 0)]; + for entity_id in entity_ids { + assert_eq!( + segment_topology(&store, tenant, entity_type, entity_id).await, + expected_topology, + "first journal append must replace legacy snapshot-only topology" + ); + assert_eq!( + event_segment_assignments(&store, tenant, entity_type, entity_id).await, + expected_assignments + ); + assert_eq!( + store + .load_snapshot(&format!("{tenant}:{entity_type}:{entity_id}")) + .await + .unwrap(), + Some((5, b"snapshot-five".to_vec())) + ); + } +} + +#[tokio::test] +async fn snapshot_ahead_of_nonempty_journal_does_not_rotate_past_the_hwm() { + let store = make_store("snapshot-ahead-of-journal").await; + let tenant = "tenant-snapshot-ahead"; + let entity_type = "Order"; + let entity_ids = ["ordinary", "batch-a", "batch-b"]; + let initial_events = vec![ + test_envelope("Created", serde_json::json!({"sequence": 1})), + test_envelope("Updated", serde_json::json!({"sequence": 2})), + ]; + + for entity_id in entity_ids { + let persistence_id = format!("{tenant}:{entity_type}:{entity_id}"); + store + .append(&persistence_id, 0, &initial_events) + .await + .unwrap(); + store + .save_snapshot(&persistence_id, 5, b"snapshot-five") + .await + .unwrap(); + assert_eq!( + segment_topology(&store, tenant, entity_type, entity_id).await, + vec![(0, 1, Some(2), None, 2, 0)], + "snapshot sequence beyond the journal HWM is not a journal boundary" + ); + seed_legacy_snapshot_ahead_segments(&store, tenant, entity_type, entity_id).await; + } + + let ordinary_id = format!("{tenant}:{entity_type}:ordinary"); store - .upsert_query_projection( - &tenant, - "Session", - "session-1", - "Completed", - &serde_json::json!({ "Name": "session" }), - 1, + .append( + &ordinary_id, + 2, + &[test_envelope("Updated", serde_json::json!({"sequence": 3}))], ) .await - .expect("upsert Session projection"); + .unwrap(); - let ids = store - .list_entity_ids_by_type(&tenant, "AgentRoute") + let batch_a_id = format!("{tenant}:{entity_type}:batch-a"); + let batch_b_id = format!("{tenant}:{entity_type}:batch-b"); + store + .append_batch(&[ + unchecked_batch_append( + &batch_a_id, + 2, + vec![test_envelope("Updated", serde_json::json!({"sequence": 3}))], + ), + unchecked_batch_append( + &batch_b_id, + 2, + vec![test_envelope("Updated", serde_json::json!({"sequence": 3}))], + ), + ]) .await - .expect("list AgentRoute IDs by type"); + .unwrap(); - assert_eq!(ids, vec!["route-main".to_string()]); + let expected_topology = vec![(0, 1, Some(3), None, 3, 0)]; + let expected_assignments = vec![(1, 0), (2, 0), (3, 0)]; + for entity_id in entity_ids { + assert_eq!( + segment_topology(&store, tenant, entity_type, entity_id).await, + expected_topology, + "append must repair a legacy future-start open segment" + ); + assert_eq!( + event_segment_assignments(&store, tenant, entity_type, entity_id).await, + expected_assignments + ); + assert_eq!( + store + .load_snapshot(&format!("{tenant}:{entity_type}:{entity_id}")) + .await + .unwrap(), + Some((5, b"snapshot-five".to_vec())) + ); + } } #[tokio::test] -async fn list_entity_ids_by_type_unions_catalog_field_index_and_events() { - let store = make_store("entity-list-by-type-union").await; - let tenant = format!("tenant-{}", uuid::Uuid::new_v4()); - +async fn append_with_index_rows_fences_absent_and_exact_snapshot_sources() { + let store = make_store("append-snapshot-source-fence").await; + let persistence_id = "tenant-fence:Order:exact-source"; store - .upsert_query_projection( - &tenant, - "AgentRoute", - "route-catalog", - "Ready", - &serde_json::json!({ "Name": "catalog" }), - 3, + .append( + persistence_id, + 0, + &[test_envelope("Created", serde_json::json!({}))], ) .await - .expect("upsert catalog projection"); + .unwrap(); store - .upsert_query_projection( - &tenant, + .save_snapshot(persistence_id, 1, b"snapshot-a") + .await + .unwrap(); + store + .save_snapshot(persistence_id, 1, b"snapshot-b") + .await + .unwrap(); + + let topology_before_reject = + segment_topology(&store, "tenant-fence", "Order", "exact-source").await; + let stale_exact = source_reconciliation(SnapshotSourceFence::Exact { + sequence_nr: 1, + state: b"snapshot-a".to_vec(), + }); + let error = store + .append_with_index_rows( + persistence_id, + 1, + &[test_envelope("Updated", serde_json::json!({}))], + &[], + &[], + stale_exact, + ) + .await + .unwrap_err(); + assert!(matches!(error, PersistenceError::SnapshotGenerationChanged)); + assert_eq!(store.read_events(persistence_id, 0).await.unwrap().len(), 1); + assert_eq!( + segment_topology(&store, "tenant-fence", "Order", "exact-source").await, + topology_before_reject + ); + + store + .append_with_index_rows( + persistence_id, + 1, + &[test_envelope("Updated", serde_json::json!({}))], + &[], + &[], + source_reconciliation(SnapshotSourceFence::Exact { + sequence_nr: 1, + state: b"snapshot-b".to_vec(), + }), + ) + .await + .unwrap(); + assert_eq!(store.read_events(persistence_id, 0).await.unwrap().len(), 2); + + let absent_id = "tenant-fence:Order:absent-source"; + store + .append_with_index_rows( + absent_id, + 0, + &[test_envelope("Created", serde_json::json!({}))], + &[], + &[], + source_reconciliation(SnapshotSourceFence::Absent), + ) + .await + .unwrap(); + store + .save_snapshot(absent_id, 1, b"appeared") + .await + .unwrap(); + let error = store + .append_with_index_rows( + absent_id, + 1, + &[test_envelope("Updated", serde_json::json!({}))], + &[], + &[], + source_reconciliation(SnapshotSourceFence::Absent), + ) + .await + .unwrap_err(); + assert!(matches!(error, PersistenceError::SnapshotGenerationChanged)); + assert_eq!(store.read_events(absent_id, 0).await.unwrap().len(), 1); +} + +#[tokio::test] +async fn checked_snapshot_save_rejects_a_changed_derivation_source() { + let store = make_store("checked-snapshot-save").await; + let tenant = "tenant-checked-snapshot"; + let entity_type = "Order"; + let entity_id = "checked-source"; + let persistence_id = format!("{tenant}:{entity_type}:{entity_id}"); + let events = (1..=4) + .map(|sequence| test_envelope("Updated", serde_json::json!({"sequence": sequence}))) + .collect::>(); + store.append(&persistence_id, 0, &events).await.unwrap(); + store + .save_snapshot(&persistence_id, 2, b"snapshot-a") + .await + .unwrap(); + + store + .save_snapshot_if_source( + &persistence_id, + 2, + b"snapshot-b", + &SnapshotSourceFence::Exact { + sequence_nr: 2, + state: b"snapshot-a".to_vec(), + }, + None, + ) + .await + .expect("current exact source may replace equal-sequence bytes"); + let topology_before_stale_write = + segment_topology(&store, tenant, entity_type, entity_id).await; + let assignments_before_stale_write = + event_segment_assignments(&store, tenant, entity_type, entity_id).await; + + let error = store + .save_snapshot_if_source( + &persistence_id, + 3, + b"snapshot-c", + &SnapshotSourceFence::Exact { + sequence_nr: 2, + state: b"snapshot-a".to_vec(), + }, + None, + ) + .await + .unwrap_err(); + assert!(matches!(error, PersistenceError::SnapshotGenerationChanged)); + assert_eq!( + store.load_snapshot(&persistence_id).await.unwrap(), + Some((2, b"snapshot-b".to_vec())) + ); + assert_eq!( + segment_topology(&store, tenant, entity_type, entity_id).await, + topology_before_stale_write + ); + assert_eq!( + event_segment_assignments(&store, tenant, entity_type, entity_id).await, + assignments_before_stale_write + ); + + let absent_id = "tenant-checked-snapshot:Order:absent-source"; + store + .save_snapshot_if_source(absent_id, 1, b"first", &SnapshotSourceFence::Absent, None) + .await + .expect("absent source may create the first snapshot"); + let error = store + .save_snapshot_if_source(absent_id, 2, b"second", &SnapshotSourceFence::Absent, None) + .await + .unwrap_err(); + assert!(matches!(error, PersistenceError::SnapshotGenerationChanged)); + assert_eq!( + store.load_snapshot(absent_id).await.unwrap(), + Some((1, b"first".to_vec())) + ); +} + +#[tokio::test] +async fn list_entity_ids_returns_distinct_pairs() { + let store = make_store("entity-list").await; + + let tenant_a = format!("tenant-a-{}", uuid::Uuid::new_v4()); + let tenant_b = format!("tenant-b-{}", uuid::Uuid::new_v4()); + + let order_1 = format!("{tenant_a}:Order:ord-1"); + let order_2 = format!("{tenant_a}:Order:ord-2"); + let task_1 = format!("{tenant_a}:Task:task-1"); + let other_tenant = format!("{tenant_b}:Order:ord-9"); + + store + .append( + &order_1, + 0, + &[test_envelope( + "OrderCreated", + serde_json::json!({ "id": "ord-1" }), + )], + ) + .await + .unwrap(); + store + .append( + &order_1, + 1, + &[test_envelope( + "OrderUpdated", + serde_json::json!({ "step": 2 }), + )], + ) + .await + .unwrap(); + store + .append( + &order_2, + 0, + &[test_envelope( + "OrderCreated", + serde_json::json!({ "id": "ord-2" }), + )], + ) + .await + .unwrap(); + store + .append( + &task_1, + 0, + &[test_envelope( + "TaskCreated", + serde_json::json!({ "id": "task-1" }), + )], + ) + .await + .unwrap(); + store + .append( + &other_tenant, + 0, + &[test_envelope( + "OrderCreated", + serde_json::json!({ "id": "ord-9" }), + )], + ) + .await + .unwrap(); + + let mut entities = store.list_entity_ids(&tenant_a).await.unwrap(); + entities.sort(); + + assert_eq!( + entities, + vec![ + ("Order".to_string(), "ord-1".to_string()), + ("Order".to_string(), "ord-2".to_string()), + ("Task".to_string(), "task-1".to_string()), + ] + ); +} + +#[tokio::test] +async fn list_entity_ids_by_type_uses_entity_catalog() { + let store = make_store("entity-list-by-type-catalog").await; + let tenant = format!("tenant-{}", uuid::Uuid::new_v4()); + + store + .upsert_query_projection( + &tenant, + "AgentRoute", + "route-main", + "Ready", + &serde_json::json!({ "Name": "main" }), + 3, + ) + .await + .expect("upsert AgentRoute projection"); + store + .upsert_query_projection( + &tenant, + "Session", + "session-1", + "Completed", + &serde_json::json!({ "Name": "session" }), + 1, + ) + .await + .expect("upsert Session projection"); + + let ids = store + .list_entity_ids_by_type(&tenant, "AgentRoute") + .await + .expect("list AgentRoute IDs by type"); + + assert_eq!(ids, vec!["route-main".to_string()]); +} + +#[tokio::test] +async fn list_entity_ids_by_type_unions_catalog_field_index_and_events() { + let store = make_store("entity-list-by-type-union").await; + let tenant = format!("tenant-{}", uuid::Uuid::new_v4()); + + store + .upsert_query_projection( + &tenant, + "AgentRoute", + "route-catalog", + "Ready", + &serde_json::json!({ "Name": "catalog" }), + 3, + ) + .await + .expect("upsert catalog projection"); + store + .upsert_query_projection( + &tenant, "AgentRoute", "route-deleted", "Ready", @@ -553,6 +1319,7 @@ async fn list_entity_ids_by_type_includes_events_and_excludes_deleted() { let deleted_order = format!("{tenant}:Order:ord-deleted"); let active_order = format!("{tenant}:Order:ord-active"); + let legacy_named_live_order = format!("{tenant}:Order:ord-legacy-named-live"); let task = format!("{tenant}:Task:task-1"); store @@ -579,6 +1346,21 @@ async fn list_entity_ids_by_type_includes_events_and_excludes_deleted() { ) .await .unwrap(); + store + .append( + &legacy_named_live_order, + 0, + &[test_envelope( + "Deleted", + serde_json::json!({ + "action": "Deleted", + "from_status": "Draft", + "to_status": "Draft" + }), + )], + ) + .await + .unwrap(); store .append(&task, 0, &[test_envelope("Created", serde_json::json!({}))]) .await @@ -589,158 +1371,700 @@ async fn list_entity_ids_by_type_includes_events_and_excludes_deleted() { .await .expect("list Order IDs by type from events"); - assert_eq!(ids, vec!["ord-active".to_string()]); + assert_eq!( + ids, + vec![ + "ord-active".to_string(), + "ord-legacy-named-live".to_string() + ] + ); + assert_eq!( + store + .terminal_tombstone_sequence(&legacy_named_live_order) + .await + .unwrap(), + None, + "structured live lifecycle metadata outranks legacy Deleted names" + ); +} + +#[tokio::test] +async fn list_entity_ids_excludes_entities_with_deleted_tombstones() { + let store = make_store("entity-list-deleted").await; + let tenant = format!("tenant-{}", uuid::Uuid::new_v4()); + + let deleted_order = format!("{tenant}:Order:ord-deleted"); + let active_order = format!("{tenant}:Order:ord-active"); + let legacy_named_live_order = format!("{tenant}:Order:ord-legacy-named-live"); + + store + .append( + &deleted_order, + 0, + &[test_envelope( + "Created", + serde_json::json!({ "id": "ord-deleted" }), + )], + ) + .await + .unwrap(); + store + .append( + &deleted_order, + 1, + &[test_envelope( + "Deleted", + serde_json::json!({ + "action": "Deleted", + "from_status": "Draft", + "to_status": "Deleted" + }), + )], + ) + .await + .unwrap(); + store + .append( + &active_order, + 0, + &[test_envelope( + "Created", + serde_json::json!({ "id": "ord-active" }), + )], + ) + .await + .unwrap(); + store + .append( + &legacy_named_live_order, + 0, + &[test_envelope( + "Deleted", + serde_json::json!({ + "action": "Deleted", + "from_status": "Draft", + "to_status": "Draft" + }), + )], + ) + .await + .unwrap(); + + let mut entities = store.list_entity_ids(&tenant).await.unwrap(); + entities.sort(); + + assert_eq!( + entities, + vec![ + ("Order".to_string(), "ord-active".to_string()), + ("Order".to_string(), "ord-legacy-named-live".to_string()) + ] + ); +} + +#[tokio::test] +async fn policy_denial_patterns_roundtrip_and_merge() { + let store = make_store("policy-denials").await; + let tenant = format!("tenant-{}", uuid::Uuid::new_v4()); + + store + .upsert_policy_denial_pattern( + &tenant, + Some("planner"), + "read", + "Issue", + "ISSUE-1", + "2026-03-23T10:00:00Z", + ) + .await + .unwrap(); + store + .upsert_policy_denial_pattern( + &tenant, + Some("planner"), + "read", + "Issue", + "ISSUE-2", + "2026-03-23T11:00:00Z", + ) + .await + .unwrap(); + + let rows = store.load_policy_denial_patterns(&tenant).await.unwrap(); + assert_eq!(rows.len(), 1); + let row = &rows[0]; + assert_eq!(row.agent_type.as_deref(), Some("planner")); + assert_eq!(row.action, "read"); + assert_eq!(row.resource_type, "Issue"); + assert_eq!(row.count, 2); + assert_eq!(row.first_seen, "2026-03-23T10:00:00Z"); + assert_eq!(row.last_seen, "2026-03-23T11:00:00Z"); + + let ids: Vec = serde_json::from_str(&row.distinct_resource_ids_json).unwrap(); + assert_eq!(ids.len(), 2); + assert!(ids.contains(&"ISSUE-1".to_string())); + assert!(ids.contains(&"ISSUE-2".to_string())); +} + +#[tokio::test] +async fn migrate_is_idempotent() { + let store = make_store("migrate-idempotent").await; + + store.migrate().await.unwrap(); + store.migrate().await.unwrap(); +} + +/// Regression: append must be durable (readable from a fresh connection) +/// before the caller receives the new sequence number. +/// +/// This is the persist-before-return ordering guarantee: the event log must +/// reflect the written event for any subsequent reader, even one that opens +/// a new connection to the same database file. +#[tokio::test] +async fn append_is_durable_before_return() { + let url = sqlite_test_url("persist-before-return"); + let store1 = TursoEventStore::new(&url, None) + .await + .expect("create store1"); + + let persistence_id = "tenant-x:Widget:w-1"; + let new_seq = store1 + .append( + persistence_id, + 0, + &[test_envelope("Created", serde_json::json!({"id": "w-1"}))], + ) + .await + .expect("append"); + + assert_eq!(new_seq, 1, "should return sequence 1 after first append"); + + // Open a new independent connection to the same DB — simulates a second + // reader or a process restart. The event must already be visible. + let store2 = TursoEventStore::new(&url, None) + .await + .expect("create store2"); + let events = store2 + .read_events(persistence_id, 0) + .await + .expect("read from second connection"); + + assert_eq!( + events.len(), + 1, + "event must be durable and readable from a fresh connection immediately after append" + ); + assert_eq!(events[0].sequence_nr, 1); + assert_eq!(events[0].event_type, "Created"); +} + +#[tokio::test] +async fn source_fenced_projection_repairs_snapshot_rewrites_and_lifecycle_races() { + let store = make_store("source-fenced-projection-races").await; + let tenant = "tenant-projection-fence"; + let entity_type = "Doc"; + let entity_id = "projection-race"; + let persistence_id = format!("{tenant}:{entity_type}:{entity_id}"); + let snapshot_a = b"snapshot-a".to_vec(); + let snapshot_b = b"snapshot-b".to_vec(); + let fields_a = serde_json::json!({"Id": entity_id, "Status": "Live", "Version": "A"}); + let state_a = fields_a.clone(); + let fields_b = serde_json::json!({"Id": entity_id, "Status": "Live", "Version": "B"}); + let state_b = fields_b.clone(); + + store + .append( + &persistence_id, + 0, + &[test_envelope( + "Created", + serde_json::json!({"to_status": "Live"}), + )], + ) + .await + .expect("seed journal generation"); + store + .save_snapshot(&persistence_id, 1, &snapshot_a) + .await + .expect("seed snapshot A"); + assert!( + store + .upsert_query_projection_with_state_if_source( + tenant, + entity_type, + entity_id, + "Live", + &fields_a, + &state_a, + 1, + ProjectionSourceFence { + expected_journal_sequence: 1, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: 1, + state: &snapshot_a, + }), + }, + ) + .await + .expect("project snapshot A") + ); + assert!( + store + .dirty_query_projection_entity_ids(tenant, entity_type, 10) + .await + .expect("read clean marker set") + .is_empty() + ); + + store + .save_snapshot(&persistence_id, 1, &snapshot_b) + .await + .expect("replace snapshot A with B at the same HWM"); + assert!( + !store + .upsert_query_projection_with_state_if_source( + tenant, + entity_type, + entity_id, + "Live", + &fields_a, + &state_a, + 1, + ProjectionSourceFence { + expected_journal_sequence: 1, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: 1, + state: &snapshot_a, + }), + }, + ) + .await + .expect("reject stale snapshot A fence") + ); + assert_eq!( + store + .dirty_query_projection_entity_ids(tenant, entity_type, 10) + .await + .expect("snapshot B remains dirty"), + vec![entity_id.to_string()] + ); + assert!( + store + .upsert_query_projection_with_state_if_source( + tenant, + entity_type, + entity_id, + "Live", + &fields_b, + &state_b, + 1, + ProjectionSourceFence { + expected_journal_sequence: 1, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: 1, + state: &snapshot_b, + }), + }, + ) + .await + .expect("repair projection from snapshot B") + ); + + // A delayed unfenced queue delivery can arrive after the repair. It may + // write its stale row, but must atomically re-mark the projection dirty so + // the next read source-fences B again instead of trusting A indefinitely. + store + .upsert_query_projection_with_state( + tenant, + entity_type, + entity_id, + "Live", + &fields_a, + &state_a, + 1, + ) + .await + .expect("deliver delayed plain projection A"); + assert_eq!( + store + .dirty_query_projection_entity_ids(tenant, entity_type, 10) + .await + .expect("delayed delivery re-marks dirty"), + vec![entity_id.to_string()] + ); + assert!( + store + .upsert_query_projection_with_state_if_source( + tenant, + entity_type, + entity_id, + "Live", + &fields_b, + &state_b, + 1, + ProjectionSourceFence { + expected_journal_sequence: 1, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: 1, + state: &snapshot_b, + }), + }, + ) + .await + .expect("repair delayed A back to B") + ); + assert!( + !store + .remove_query_projection_if_exact( + tenant, + entity_type, + entity_id, + "Live", + &fields_a, + &state_a, + 1, + ) + .await + .expect("full-row cleanup CAS preserves B") + ); + let row = store + .load_entity_catalog_rows(tenant, entity_type, &[entity_id.to_string()]) + .await + .expect("load repaired B row") + .pop() + .expect("B row remains"); + assert_eq!(row.fields, fields_b); + assert!( + store + .query_field_index( + tenant, + entity_type, + "field_name = ?3 AND field_value = ?4", + vec!["Version".to_string(), "B".to_string()], + ) + .await + .expect("query B field index") + .contains(&entity_id.to_string()), + "a failed exact cleanup must preserve B's EAV rows" + ); + + store + .append( + &persistence_id, + 1, + &[test_envelope( + "Delete", + serde_json::json!({"to_status": "Deleted"}), + )], + ) + .await + .expect("advance source to Deleted"); + assert!( + !store + .remove_query_projection_if_source( + tenant, + entity_type, + entity_id, + ProjectionSourceFence { + expected_journal_sequence: 1, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: 1, + state: &snapshot_b, + }), + }, + ) + .await + .expect("reject stale live removal fence") + ); + assert!( + store + .remove_query_projection_if_source( + tenant, + entity_type, + entity_id, + ProjectionSourceFence { + expected_journal_sequence: 2, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: 1, + state: &snapshot_b, + }), + }, + ) + .await + .expect("remove Deleted projection") + ); + assert!( + store + .load_entity_catalog_rows(tenant, entity_type, &[entity_id.to_string()]) + .await + .expect("load removed catalog") + .is_empty() + ); + + store + .append( + &persistence_id, + 2, + &[test_envelope( + "Restore", + serde_json::json!({"to_status": "Live"}), + )], + ) + .await + .expect("advance source back to Live"); + assert!( + store + .upsert_query_projection_with_state_if_source( + tenant, + entity_type, + entity_id, + "Live", + &fields_b, + &state_b, + 3, + ProjectionSourceFence { + expected_journal_sequence: 3, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: 1, + state: &snapshot_b, + }), + }, + ) + .await + .expect("restore Live projection") + ); + assert!( + store + .dirty_query_projection_entity_ids(tenant, entity_type, 1) + .await + .expect("restored source is clean") + .is_empty() + ); + + // If source closing fails after a conditional upsert, exact cleanup removes + // the attempted row. With no concurrent source writer, cleanup itself must + // leave a durable marker so a later read rebuilds the missing projection. + assert!( + store + .remove_query_projection_if_exact( + tenant, + entity_type, + entity_id, + "Live", + &fields_b, + &state_b, + 3, + ) + .await + .expect("remove exact attempted projection after closing fault") + ); + assert_eq!( + store + .dirty_query_projection_entity_ids(tenant, entity_type, 1) + .await + .expect("exact cleanup marks projection dirty"), + vec![entity_id.to_string()] + ); + assert!( + store + .upsert_query_projection_with_state_if_source( + tenant, + entity_type, + entity_id, + "Live", + &fields_b, + &state_b, + 3, + ProjectionSourceFence { + expected_journal_sequence: 3, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: 1, + state: &snapshot_b, + }), + }, + ) + .await + .expect("repair exact-cleanup absence") + ); + assert!( + store + .dirty_query_projection_entity_ids(tenant, entity_type, 1) + .await + .expect("exact-cleanup repair clears marker") + .is_empty() + ); } #[tokio::test] -async fn list_entity_ids_excludes_entities_with_deleted_tombstones() { - let store = make_store("entity-list-deleted").await; - let tenant = format!("tenant-{}", uuid::Uuid::new_v4()); - - let deleted_order = format!("{tenant}:Order:ord-deleted"); - let active_order = format!("{tenant}:Order:ord-active"); +async fn dirty_projection_upgrade_seed_covers_sources_and_catalog_only_rows_once() { + let url = sqlite_test_url("projection-dirty-upgrade-seed"); + let store = TursoEventStore::new(&url, None) + .await + .expect("create pre-upgrade store"); + let tenant = "tenant-upgrade-seed"; + let entity_type = "Doc"; + let source_id = "source-backed"; + let source_pid = format!("{tenant}:{entity_type}:{source_id}"); + let catalog_only_id = "catalog-only"; store .append( - &deleted_order, + &source_pid, 0, &[test_envelope( "Created", - serde_json::json!({ "id": "ord-deleted" }), + serde_json::json!({"to_status": "Live"}), )], ) .await - .unwrap(); + .expect("seed authoritative source"); store - .append( - &deleted_order, - 1, - &[test_envelope( - "Deleted", - serde_json::json!({ - "action": "Deleted", - "from_status": "Draft", - "to_status": "Deleted" - }), - )], + .upsert_query_projection( + tenant, + entity_type, + catalog_only_id, + "Live", + &serde_json::json!({"Id": catalog_only_id, "Status": "Live"}), + 0, ) .await - .unwrap(); + .expect("seed catalog-only compatibility row"); + store - .append( - &active_order, - 0, - &[test_envelope( - "Created", - serde_json::json!({ "id": "ord-active" }), - )], - ) + .connection() + .expect("open migration connection") + .execute("DROP TABLE query_projection_dirty", ()) .await - .unwrap(); + .expect("simulate database created before dirty-ledger migration"); - let mut entities = store.list_entity_ids(&tenant).await.unwrap(); - entities.sort(); + let interrupted_connection = store.connection().expect("open interrupted migration"); + let interrupted = interrupted_connection + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .expect("begin interrupted migration"); + interrupted + .execute(crate::schema::CREATE_QUERY_PROJECTION_DIRTY_TABLE, ()) + .await + .expect("create ledger inside interrupted migration"); + interrupted + .rollback() + .await + .expect("simulate interruption before seed publication"); + let upgraded = TursoEventStore::new(&url, None) + .await + .expect("run dirty-ledger upgrade"); assert_eq!( - entities, - vec![("Order".to_string(), "ord-active".to_string())] + upgraded + .dirty_query_projection_entity_ids(tenant, entity_type, 10) + .await + .expect("read upgrade seed"), + vec![catalog_only_id.to_string(), source_id.to_string()], + "the one-time seed must gate both missing source projections and pre-existing catalog rows" ); -} -#[tokio::test] -async fn policy_denial_patterns_roundtrip_and_merge() { - let store = make_store("policy-denials").await; - let tenant = format!("tenant-{}", uuid::Uuid::new_v4()); + assert!( + upgraded + .clear_query_projection_dirty_if_source( + tenant, + entity_type, + catalog_only_id, + ProjectionSourceFence { + expected_journal_sequence: 0, + expected_snapshot: None, + }, + ) + .await + .expect("acknowledge catalog-only row") + ); + assert_eq!( + upgraded + .load_entity_catalog_rows(tenant, entity_type, &[catalog_only_id.to_string()]) + .await + .expect("load catalog-only row after acknowledgement") + .len(), + 1, + "upgrade repair must preserve catalog-only compatibility rows" + ); - store - .upsert_policy_denial_pattern( - &tenant, - Some("planner"), - "read", - "Issue", - "ISSUE-1", - "2026-03-23T10:00:00Z", - ) - .await - .unwrap(); - store - .upsert_policy_denial_pattern( - &tenant, - Some("planner"), - "read", - "Issue", - "ISSUE-2", - "2026-03-23T11:00:00Z", - ) + let reopened = TursoEventStore::new(&url, None) .await - .unwrap(); - - let rows = store.load_policy_denial_patterns(&tenant).await.unwrap(); - assert_eq!(rows.len(), 1); - let row = &rows[0]; - assert_eq!(row.agent_type.as_deref(), Some("planner")); - assert_eq!(row.action, "read"); - assert_eq!(row.resource_type, "Issue"); - assert_eq!(row.count, 2); - assert_eq!(row.first_seen, "2026-03-23T10:00:00Z"); - assert_eq!(row.last_seen, "2026-03-23T11:00:00Z"); - - let ids: Vec = serde_json::from_str(&row.distinct_resource_ids_json).unwrap(); - assert_eq!(ids.len(), 2); - assert!(ids.contains(&"ISSUE-1".to_string())); - assert!(ids.contains(&"ISSUE-2".to_string())); -} - -#[tokio::test] -async fn migrate_is_idempotent() { - let store = make_store("migrate-idempotent").await; - - store.migrate().await.unwrap(); - store.migrate().await.unwrap(); + .expect("reopen upgraded store"); + assert_eq!( + reopened + .dirty_query_projection_entity_ids(tenant, entity_type, 10) + .await + .expect("read ledger after ordinary reopen"), + vec![source_id.to_string()], + "an ordinary restart must not re-seed already acknowledged catalog-only rows" + ); } -/// Regression: append must be durable (readable from a fresh connection) -/// before the caller receives the new sequence number. -/// -/// This is the persist-before-return ordering guarantee: the event log must -/// reflect the written event for any subsequent reader, even one that opens -/// a new connection to the same database file. #[tokio::test] -async fn append_is_durable_before_return() { - let url = sqlite_test_url("persist-before-return"); - let store1 = TursoEventStore::new(&url, None) - .await - .expect("create store1"); +async fn exact_snapshot_source_replaces_a_higher_stale_catalog_sequence() { + let store = make_store("snapshot-source-replaces-stale-catalog").await; + let tenant = "tenant-snapshot-catalog-fence"; + let entity_type = "Doc"; + let entity_id = "snapshot-owner"; + let persistence_id = format!("{tenant}:{entity_type}:{entity_id}"); + let stale_fields = serde_json::json!({"Id": entity_id, "Status": "Live", "Version": "stale"}); + let current_fields = + serde_json::json!({"Id": entity_id, "Status": "Live", "Version": "snapshot"}); + let snapshot = b"authoritative-snapshot".to_vec(); - let persistence_id = "tenant-x:Widget:w-1"; - let new_seq = store1 - .append( - persistence_id, - 0, - &[test_envelope("Created", serde_json::json!({"id": "w-1"}))], + store + .upsert_query_projection_with_state( + tenant, + entity_type, + entity_id, + "Live", + &stale_fields, + &stale_fields, + 10, ) .await - .expect("append"); - - assert_eq!(new_seq, 1, "should return sequence 1 after first append"); - - // Open a new independent connection to the same DB — simulates a second - // reader or a process restart. The event must already be visible. - let store2 = TursoEventStore::new(&url, None) - .await - .expect("create store2"); - let events = store2 - .read_events(persistence_id, 0) + .expect("seed higher catalog-only compatibility sequence"); + store + .save_snapshot(&persistence_id, 5, &snapshot) .await - .expect("read from second connection"); - - assert_eq!( - events.len(), - 1, - "event must be durable and readable from a fresh connection immediately after append" + .expect("publish lower authoritative snapshot generation"); + assert!( + store + .upsert_query_projection_with_state_if_source( + tenant, + entity_type, + entity_id, + "Live", + ¤t_fields, + ¤t_fields, + 5, + ProjectionSourceFence { + expected_journal_sequence: 0, + expected_snapshot: Some(SnapshotBackfillFence { + sequence_nr: 5, + state: &snapshot, + }), + }, + ) + .await + .expect("repair from exact snapshot source"), + "the exact current snapshot must outrank a higher stale catalog sequence" + ); + let row = store + .load_entity_catalog_rows(tenant, entity_type, &[entity_id.to_string()]) + .await + .expect("load repaired catalog") + .pop() + .expect("repaired catalog row"); + assert_eq!(row.sequence_nr, 5); + assert_eq!(row.fields, current_fields); + assert!( + store + .dirty_query_projection_entity_ids(tenant, entity_type, 1) + .await + .expect("read closed dirty ledger") + .is_empty() ); - assert_eq!(events[0].sequence_nr, 1); - assert_eq!(events[0].event_type, "Created"); } #[tokio::test] @@ -765,6 +2089,14 @@ async fn query_projection_roundtrip_updates_catalog_and_field_index() { ) .await .expect("upsert query projection"); + assert!( + store + .dirty_query_projection_entity_ids(&tenant, entity_type, 1) + .await + .expect("read catalog-only dirty ledger") + .is_empty(), + "a projection with no journal or snapshot source remains a supported catalog-only row" + ); let title_matches = store .query_field_index( @@ -787,6 +2119,14 @@ async fn query_projection_roundtrip_updates_catalog_and_field_index() { .remove_query_projection(&tenant, entity_type, entity_id) .await .expect("remove query projection"); + assert!( + store + .dirty_query_projection_entity_ids(&tenant, entity_type, 1) + .await + .expect("read catalog-only dirty ledger after removal") + .is_empty(), + "removing a catalog-only row must not create an unrecoverable dirty marker" + ); let remaining = store .query_field_index( @@ -1017,6 +2357,161 @@ async fn query_projection_batch_updates_catalog_and_field_index() { assert_eq!(rows[1].sequence_nr, 3); } +#[tokio::test] +async fn journal_generation_replaces_a_numerically_newer_migration_projection() { + let store = make_store("journal-generation-replaces-migration-projection").await; + let tenant = format!("tenant-{}", uuid::Uuid::new_v4()); + let entity_type = "Order"; + let entity_id = "ord-materialized"; + let persistence_id = format!("{tenant}:{entity_type}:{entity_id}"); + + let migration_fields = serde_json::json!({"Title": "migration"}); + let migration_state = serde_json::json!({ + "entity_type": entity_type, + "entity_id": entity_id, + "status": "Draft", + "fields": migration_fields, + "sequence_nr": 5, + }); + store + .upsert_query_projection_with_state( + &tenant, + entity_type, + entity_id, + "Draft", + migration_state.get("fields").unwrap(), + &migration_state, + 5, + ) + .await + .expect("seed migration projection"); + + store + .append( + &persistence_id, + 0, + &[ + test_envelope( + "Temper.Internal.StateMaterialization.v1", + serde_json::json!({}), + ), + test_envelope("Touch", serde_json::json!({})), + ], + ) + .await + .expect("seed materialized journal generation"); + + let journal_fields = serde_json::json!({"Title": "journal"}); + let journal_state = serde_json::json!({ + "entity_type": entity_type, + "entity_id": entity_id, + "status": "Ready", + "fields": journal_fields, + "sequence_nr": 2, + }); + store + .upsert_query_projection_with_state( + &tenant, + entity_type, + entity_id, + "Ready", + journal_state.get("fields").unwrap(), + &journal_state, + 2, + ) + .await + .expect("write current journal projection"); + + let rows = store + .load_entity_catalog_rows(&tenant, entity_type, &[entity_id.to_string()]) + .await + .expect("load current projection"); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].sequence_nr, 2); + assert_eq!(rows[0].fields["Title"], "journal"); + assert_eq!( + store + .query_field_index( + &tenant, + entity_type, + "field_name = ?3 AND field_value = ?4", + vec!["Title".to_string(), "journal".to_string()], + ) + .await + .expect("query current journal projection"), + vec![entity_id.to_string()] + ); + assert!( + store + .query_field_index( + &tenant, + entity_type, + "field_name = ?3 AND field_value = ?4", + vec!["Title".to_string(), "migration".to_string()], + ) + .await + .expect("query retired migration projection") + .is_empty() + ); + + for (delayed_sequence, title) in [(5, "delayed-migration"), (1, "stale-journal")] { + let delayed_fields = serde_json::json!({"Title": title}); + let delayed_state = serde_json::json!({ + "entity_type": entity_type, + "entity_id": entity_id, + "status": "Delayed", + "fields": delayed_fields, + "sequence_nr": delayed_sequence, + }); + store + .upsert_query_projection_with_state( + &tenant, + entity_type, + entity_id, + "Delayed", + delayed_state.get("fields").unwrap(), + &delayed_state, + delayed_sequence, + ) + .await + .expect("ignore projection outside the current journal generation"); + } + + let rows = store + .load_entity_catalog_rows(&tenant, entity_type, &[entity_id.to_string()]) + .await + .expect("reload current projection"); + assert_eq!(rows[0].sequence_nr, 2); + assert_eq!(rows[0].status, "Ready"); + assert_eq!(rows[0].fields["Title"], "journal"); + assert_eq!( + store + .query_field_index( + &tenant, + entity_type, + "field_name = ?3 AND field_value = ?4 AND status = 'Ready'", + vec!["Title".to_string(), "journal".to_string()], + ) + .await + .expect("query journal projection after rejected updates"), + vec![entity_id.to_string()] + ); + for title in ["delayed-migration", "stale-journal"] { + assert!( + store + .query_field_index( + &tenant, + entity_type, + "field_name = ?3 AND field_value = ?4", + vec!["Title".to_string(), title.to_string()], + ) + .await + .expect("query rejected projection value") + .is_empty() + ); + } +} + #[tokio::test] async fn query_projection_batch_can_store_fields_without_indexing_them() { let store = make_store("query-projection-batch-index-subset").await; diff --git a/docs/adrs/0077-catalog-first-odata-materialization.md b/docs/adrs/0077-catalog-first-odata-materialization.md index 1e5ddba12..396a16fcc 100644 --- a/docs/adrs/0077-catalog-first-odata-materialization.md +++ b/docs/adrs/0077-catalog-first-odata-materialization.md @@ -68,6 +68,17 @@ No data migration required for new deployments — the projection is populated o For deployments that were migrated from another backend (e.g., Turso → Postgres ETL): the ETL must populate `entity_catalog` directly from `snapshots`, since migrated snapshots never replay through the live transition path. The openpaw 2026-04-30 cutover discovered this gap and backfilled `entity_catalog` (1287 rows) and `entity_field_index` (11363 rows) post-hoc; the methodology is captured in ADR-0074. +Derived-index repair must consequently treat snapshots, `entity_catalog`, and +`entity_field_index` as durable entity-enumeration sources rather than assuming an +event stream exists. A catalog-only row can reconstruct declared keys; a durable row +that cannot be reconstructed must prevent authoritative coverage from publishing. +Before declared-key coverage exists, exact-key scans may materialize such a catalog +row only after proving that the entity's journal is absent both before and after the +read. Once coverage exists, the key row's sequence must equal the catalog row's +sequence, and a stable ownership row plus reconciliation revision fences the result. +Journal-backed entities continue to materialize from journal state, so a lagging live +catalog row can never outrank a tombstone. + ## Alternatives considered - **Vercel ISR + revalidate** on katagami: caches the SSR'd HTML at the edge for N seconds. Mitigation, not a root fix; first request after revalidation still pays the 22s cost. Composes with this ADR but does not replace it. diff --git a/docs/adrs/0192-exact-declared-key-reconciliation.md b/docs/adrs/0192-exact-declared-key-reconciliation.md new file mode 100644 index 000000000..caeb86bba --- /dev/null +++ b/docs/adrs/0192-exact-declared-key-reconciliation.md @@ -0,0 +1,487 @@ +# ADR-0192: Exact Declared-Key and Durable-Projection Reconciliation + +- Status: Proposed +- Date: 2026-07-13 +- Deciders: Temper core maintainers +- Related: + - ADR-0077: Catalog-only migration compatibility + - ADR-0153: Declared composite-key index + - ADR-0155: Declared vector access path + - ARN-238: Stale declared-key ownership after delete or key removal + - `crates/temper-runtime/src/persistence/` + - `crates/temper-server/src/entity_actor/` + - `crates/temper-server/src/odata/query_plane_read/` + - `crates/temper-server/src/state/projection_backfill/` + - `crates/temper-store-{postgres,sim,turso,redis}` + +## Context + +ADR-0153 co-committed declared-key rows with journal appends, but the persistence +contract could not distinguish “this caller does not maintain keys” from “this +keyed entity now owns the empty set.” Stores removed prior rows only for key names +present in the new set. A deleted entity, an all-null key, a removed declaration, +or a non-scalar key could therefore retain ownership indefinitely. + +Repair also treated durable state as a single sequence. That is insufficient for +the data already supported by Temper: + +- a snapshot-only migration generation and its first journal generation can use + the same numeric sequence; +- a snapshot can be replaced at the same sequence with different bytes; +- a numerically newer stale snapshot or catalog row can coexist with an older + terminal journal tombstone; +- a journal can be longer than the in-memory replay-tail budget; +- a projection writer can arrive after a source repair and overwrite it; +- a crash can land between an authoritative source write and its asynchronous + catalog/EAV projection. + +Those cases make key ownership, query projections, and actor recovery one source- +authority problem. Fixing only the key delete would allow a stale projection or +snapshot to reintroduce the same invalid state on the next repair. + +## Decision + +### 1. Declared keys use explicit exact-set ownership + +`IndexReconciliation` carries an explicit `keys` signal. Participating callers +pass the entity's complete post-write `key_rows`, including an empty set. Plain +`append` does not reconcile keys; `append_with_keys` and keyed actor/composite +writes do. `PersistenceAppend` carries the same `reconcile_keys` decision for +atomic multi-stream writes. + +Every durable write surface derives the final declared-key set: normal actions, +PATCH/PUT field updates, data-only creates, File initial-content writes, and +composite batches. A deleted entity and an all-null or otherwise unkeyable entity +participate with an authoritative empty set. `known_new` remains only a planning +hint; stores still validate the durable stream and key contract. + +PostgreSQL validates claims, appends the journal records, deletes every prior row +for the participating entity, inserts the complete new set, and commits once. A +batch validates all streams and claims before publishing any deletion, permitting +an atomic ownership transfer. Sim constructs and validates the complete next key +map under its deterministic lock before mutating journals or ownership. Any +conflict rolls back the whole operation. + +Turso remains non-authoritative for declared keys under ADR-0153 because it cannot +co-commit uniqueness with the journal. The capability boundary prevents its legacy +rows or watermarks from becoming authoritative hits or misses. + +**Why this approach**: row presence cannot encode an authoritative empty set. +Deleting by entity identity also removes rows for declarations that no longer +exist, without sentinel values or store knowledge of the spec. + +### 2. Actor recovery captures a complete durable source generation + +Recovery captures all of the following before replay and verifies the same values +again before returning: + +- the journal boundary, including its inclusive high-water and first terminal + sequence; +- snapshot presence, sequence, and exact serialized bytes; +- the transition table used to replay the state. + +Journal records are read in bounded ascending pages through the captured high- +water. Every page must contain the exact contiguous range requested. A short, +gapped, failed, truncated, or undecodable page is uncertainty and fails closed. +Paging bounds allocation without imposing the 10,000-event in-memory tail budget +on a valid long-lived stream. + +Once a journal exists, it owns lifecycle even when a stale snapshot has a larger +numeric sequence. A valid snapshot may provide legacy baseline fields, but journal +events overlay it and a terminal tombstone remains terminal through the captured +high-water. Snapshot-only state is authoritative only while the journal is empty. +Snapshot state is normalized and validated before use: top-level entity type/id and +status must match the requested entity and transition table, and `fields.Id` and +`fields.Status` are canonicalized from that validated identity. `Deleted` remains a +valid runtime terminal marker even though it is not a user-declared IOA state. + +A snapshot whose provenance is proven against the captured journal is an exact +replay baseline: recovery replays only the contiguous journal tail after its +sequence. Replaying its covered prefix would double-apply relative updates and +regenerate persisted nondeterministic effect values. A legacy snapshot without +journal provenance is instead a migration input; recovery strictly replays the +captured journal lifecycle so an older terminal record cannot be hidden by newer +stale snapshot bytes. + +The first journal write from a snapshot-only stream atomically appends a versioned +state materialization record followed by the requested domain record. The control +record is recognized only when both its event type and its full schema/payload +shape match, including identity, state coordinates, empty recent events, and the +bounded idempotency map. A legal domain action named +`Temper.Internal.StateMaterialization.v1` with any other payload remains a normal +domain event. + +The event store validates the exact snapshot fence, commits the materialization +and domain records, and retires the snapshot in the same transaction. A delayed +snapshot writer is suppressed after a valid materialization record is durable. +This establishes an unambiguous journal generation without losing the snapshot +baseline. Composite batches use the same handoff for every snapshot-only stream. + +Optimistic-concurrency recovery rebuilds from a clean source. It then re-checks +the durable idempotency map before re-evaluating an action, so a stale replica that +recovers the winning action returns that committed result instead of appending it +again. Actor bootstrap requires both `sequence_nr == 0` and +`total_event_count == 0`; a materialization followed only by a composite audit +record can therefore restart without fabricating a `Created` domain event. + +Ordinary hydration permits an empty-range journal read failure only when the +captured boundary proves that no journal record exists. Any non-empty boundary, +source change, page failure, or decode uncertainty fails closed. This deliberately +trades availability for durable-source correctness. + +**Why this approach**: numeric “newest wins” cannot distinguish an equal-sequence +source replacement and can resurrect a stale live snapshot over a terminal journal. +The materialization record makes snapshot retirement replayable instead of relying +on an out-of-band assumption. + +### 3. Key backfill is exact, source-fenced, and contract-revision fenced + +The repair universe is the union of events, snapshots, catalog rows, field-index +rows, and existing key rows. For each entity, repair reconstructs one stable source +generation, derives the complete current key set, and conditionally replaces the +entity's rows only while all captured facts remain true: + +- target key-set signature and monotonic contract revision; +- exact journal boundary and terminal classification; +- exact snapshot presence, sequence, and bytes; +- current stream liveness and reconstruction coordinates. + +A changed snapshot, append, tombstone, recreate, contract change, or concurrent +claim rejects the row repair. The next pass starts from current state. A terminal +journal tombstone outranks any later stale live snapshot/catalog. Catalog-only +state is accepted solely when both journal and snapshot are absent; field-index- +only state cannot reconstruct a complete entity and fails the type closed. + +Backfill establishes its target signature before replay. Every source mutation +that affects reconciliation advances the type revision and removes the watermark; +each row repair and final publication compares both signature and revision. The +persisted signature includes the new derivation-contract version, forcing every +older keyed type through one complete pass. A conflicting durable claim is not +skipped: it prevents watermark publication until the underlying data is repaired. + +**Why this approach**: a SQL-only cleanup cannot replay domain state, and checking +only at final watermark publication cannot undo stale row mutations already made +during a crossed pass. + +### 4. Query projections have a durable dirty-source protocol + +`entity_catalog` and `entity_field_index` are derived state. PostgreSQL and Turso +therefore maintain `query_projection_dirty(tenant, entity_type, entity_id)`: + +1. A journal or snapshot mutation marks the entity dirty in the same transaction. +2. A projection repair carries an exact `ProjectionSourceFence`: journal high-water + plus snapshot presence, sequence, and exact bytes. +3. A source-fenced upsert, delete, or catalog-only acknowledgement clears the marker + in the same transaction only when the fence still matches. +4. An unfenced projection write for a source-backed entity re-marks it dirty. This + includes a delayed old asynchronous delivery after a successful repair. +5. Exact cleanup of an unstable attempted row compares the full catalog row + (status, fields, state, sequence). If it removes the row it marks dirty, so a + later repair restores it; a concurrent newer catalog/EAV row is preserved. + +After an exact source match, the source generation outranks the prior catalog +sequence. This lets a snapshot-only sequence 5 repair replace a stale compatibility +catalog sequence 10. Sequence monotonicity still rejects unfenced stale deliveries. + +Catalog-only ADR-0077 rows remain supported. When no journal or snapshot exists, +repair clears the seeded marker without deleting the row; ordinary unfenced writes +to such a row do not create an unrecoverable marker. + +Before a native OData read trusts catalog or EAV data, it enumerates at most +`repair_budget + 1` dirty IDs, repairs one bounded batch, then checks the ledger +again. If work remains it returns a retryable `503 ProjectionUnstable`, but the +completed batch stays repaired. Repeated reads therefore make monotonic bounded +progress rather than failing forever before doing work. + +The PostgreSQL migration creates the ledger, installs event/snapshot/catalog +mutation triggers, and seeds the union of existing events, snapshots, and catalog +rows in one migration transaction. The triggers bridge source writes and delayed +projectors from an older binary between migration and process drain. Turso checks +for the ledger, creates it, creates its index, and performs the same one-time union +seed in one `Immediate` transaction; a crash before commit leaves no partially +published table, so startup retries the whole migration. + +**Why this approach**: a source fence prevents a stale repair, while the durable +marker prevents a source write or delayed projection from becoming a permanently +silent false negative after a crash. + +### 5. Reads give co-committed ownership precedence + +An exact declared-key filter consults the key index only after the current contract +signature is fully published. Coverage and lookup are fenced by the monotonic +contract revision. A revision change discards the proof and falls back to the +bounded authoritative scan. + +Before coverage completes, neither an old hit nor an old miss is authoritative. +Recognized key queries bypass asynchronous projection authority and scan the stable +journal/snapshot sources within the configured budget. A large incomplete type may +return 413; an unstable source or dirty-projection repair may return 503. Neither is +converted into a false-empty success. Replayed `Deleted` state is absent from every +read mode and schedules idempotent projection removal. + +Even under complete coverage, a key-row hit is closed against the recovered fields, +not only its numeric sequence. Same-sequence snapshot replacement can preserve the +row's sequence while changing its declared-key values; such a mismatch is unstable +ownership and cannot certify the stale owner. + +**Why this approach**: key ownership and journal state co-commit; catalog/EAV data +converges later. The stronger source must decide identity and liveness. + +### 6. Tenant runtime generations publish behind one admission barrier + +A tenant's specs, CSDL, cross-invariants, declared-key activation epochs, Cedar +policies, reaction graph, WASM name-to-hash mappings, OS-app content, seed data, +and in-progress installed-app publication record form one runtime generation. +In the single-process runtime they share a per-tenant asynchronous read/write +barrier: + +- admitted requests retain a read lease for the complete dispatch lifetime; +- a publication acquires the write side without unbounded waiting; +- arming the writer evicts resident actors and closes direct actor resolution; +- the writer persists first, activates every in-memory component, creates required + app content and seed entities, reopens the tenant, and schedules post-cutover + reconciliation before any cancellable acknowledgement wait; +- terminal `installed` metadata is a derived recovery marker written after that + cutover, so a lost acknowledgement or cancellation cannot retain an + unreconstructible sticky intent; and +- completion advances a monotonic in-memory generation number. + +Arming also records a content-derived publication intent and sticky debt. Once a +durable mutation begins, cancellation, an error, or a lost acknowledgement cannot +prove that nothing committed. The tenant therefore remains gated after the writer +unwinds. Only a serialized retry of the exact same intent may discharge that debt; +an unrelated or partially matching publication cannot reopen traffic. + +The publication workflow itself needs to spawn and dispatch actors after the new +registry is installed but before external traffic resumes. It receives a released +`TenantGenerationLease` bound to the writer's tenant and captured generation. +Actor resolution accepts that context only while the tenant is gated at exactly +that generation, and it never bypasses an unfinished declared-key activation. +Unscoped callers and contexts from retired generations remain fenced. Scheduled +actions, state timeouts, compensations, reactions, and protocol streams retain or +capture their originating generation; detached work is discarded rather than +executed against a replacement generation. + +Direct WASM mutations authorize both before and after acquiring the writer, so a +policy cutover cannot be crossed between authorization and publication. WASM reads +hold the generation reader across authorization and registry/history access. + +This barrier is intentionally process-local. The supported runtime topology for +this release is one server process per tenant store. A multi-process runtime must +replace it with a durable distributed generation lease before horizontal writers +are enabled. + +**Why this approach**: individually atomic database rows do not prevent a request +from mixing old actors or authorization with new specs, reactions, or modules. +One admission boundary makes the complete runtime generation the unit of visibility. + +### 7. Granular policy ownership and Genesis provenance join OS-app publication + +Granular Cedar rows are the canonical durable policy source. An OS app owns a +stable row set identified by `created_by = "os-app:"`; its publication +atomically replaces that owner's complete set while preserving other owners. The +legacy aggregate tenant-policy blob is regenerated in the same transaction as a +read cache. Startup reconstructs from granular rows when they exist, so a crash +cannot promote a partial row-by-row update or revive a removed app policy. + +Governance-decision policy effects publish one stable decision-owned row through +the same durable policy generation helper. The post-action hook is told when the +API already performed that publication and must not create a second authority. +Persistence failures remain request failures rather than successful but +non-durable approvals. + +Genesis resolves the complete pinned dependency closure before installation and +constructs one `InstalledAppRecord` per resolved app. Its source kind, registry, +pinned/current hashes, follow policy, and closure identifier are passed into the +same atomic OS-app publication transaction; they are not patched after cutover. +All closure members reconcile serially under one retained tenant publication +writer, and every supplied provenance record must be consumed by that closure. + +**Why this approach**: policy text, policy ownership, and installation provenance +decide what can run and what can be restored after restart. Post-cutover repair of +either is an observable split generation, not harmless metadata enrichment. + +### 8. Composite retries use a content-bound durable batch claim + +Composite dispatch no longer proves idempotency by scanning an unbounded parent +journal. The deterministic first append carries a `PersistenceBatchIdempotency` +claim containing the parent persistence id, caller idempotency key, and a digest of +the complete parent audit plus composite sub-write intent. The store checks and +co-commits that claim with the batch: + +- the same key and digest returns the committed result without reapplying staged + projections; +- the same key with different content is rejected; and +- a failed or conflicting batch publishes neither events, ownership, nor a claim. + +Sim, PostgreSQL, Turso, and the supported single-stream Redis batch path implement +the same contract. PostgreSQL migration 0015 and Turso's transactional schema add +the durable claim table; Redis uses a namespaced claim key in the append Lua +transaction. A partially generated pack object retains its existing repair +capability by using an intent-qualified claim until the complete payload is ready. + +Within the current single-process tenant runtime, callbacks sharing the exact raw +claim namespace are serialized from claim inspection through append by a bounded +weak lock registry. This closes the pre-append race in which both callbacks could +observe no claim and one would surface an optimistic conflict instead of replaying +the other's committed result. The durable store remains the final authority, and +different claim namespaces remain concurrent. A multi-process runtime must replace +this local admission serializer with a durable lease or reservation protocol. + +**Why this approach**: bounded replay tails and journal paging are recovery tools, +not an idempotency index. A small co-committed claim makes retry cost independent of +stream age without weakening content binding. + +### 9. Materialization and reconciliation consume explicit budgets + +The snapshot-to-journal materialization envelope is bounded twice: its recovered +idempotency map retains only the deterministic newest entry budget, and the exact +borrowed reset-state representation must serialize within a 16 MiB payload budget +before any large clone or append occurs. The final candidate is checked again after +canonical `Id` and `Status` insertion. + +Key reconciliation enumerates the durable union through a stable entity-id cursor +in pages of at most 256 and yields between entities. Coverage is published only +after every page reaches an empty successor. Redis journal-boundary compatibility +reconstruction likewise scans a captured high-water in bounded pages, validates +contiguous sequences, and installs the first terminal sequence with a high-water +CAS. Later boundary reads are constant-time, and a terminal record continues to +outrank any legacy live suffix. + +**Why this approach**: replacing an unbounded allocation with a silent fixed prefix +would certify incomplete state. Explicit page and byte budgets retain progress while +making truncation or oversized state a visible failure. + +## Rollout Plan + +This release introduces a durable control record that older readers cannot replay. +It is therefore a drain-and-cutover release, not an ordinary overlapping rolling +deployment. + +1. **Install the PostgreSQL migrations first** — migration 0014 atomically installs + the dirty ledger, mixed-version source/catalog triggers, and upgrade seed; + migration 0015 adds monotonic key-contract activation state and durable composite + batch-idempotency claims. Old writers may continue briefly; the 0014 triggers keep + their source and delayed projector writes dirty. +2. **Drain and stop every old reader, writer, and projector** — verify no old binary + can serve or mutate an entity before enabling the new actor code. The triggers + cover writes during the drain, but they cannot teach an old reader to interpret + a state materialization record. +3. **Start only the new binary** — Turso deployments stop the single old process, + run the atomic local migration on open, then start serving. +4. **Repair before authority** — dirty projection reads drain bounded batches, and + keyed types remain scan-safe until their new contract watermark publishes. +5. **Observe** — monitor projection-unstable responses, dirty-ledger depth, key + repair failures, conflicts, and coverage publication before declaring the + cutover complete. + +## Readiness Gates + +- RED/GREEN history preserves the original delete/null-key ownership regression. +- Actor and composite regressions cover snapshot materialization, exact retirement, + same-name domain-event collision, stale-replica idempotency, composite-audit-only + restart, and snapshot identity/status normalization. +- Seeded DST covers tombstone precedence, paged high-water replay, truncation, + equal-sequence source replacement, source/liveness faults, and reclaim races. +- PostgreSQL and Turso tests cover exact snapshot-byte fences, lower authoritative + snapshot over higher stale catalog, delayed unfenced projection delivery, full-row + cleanup CAS, Live/Deleted/Live transitions, catalog-only acknowledgement, migration + seeding, and mixed-version PostgreSQL triggers. +- A bounded dirty set larger than the read budget proves first-request progress and + next-request completion. +- More than one 256-entity key-repair page must reconcile before coverage publishes; + a pre-metadata Redis stream must recover a terminal boundary beyond its first + 1,024-event page and retain the earlier tombstone over a legacy suffix. +- Publication regressions prove that unscoped actor resolution is closed while the + writer is armed, only the exact current internal context may bootstrap content, + retired timers cannot dispatch, and WASM reads/writes cannot cross generations. +- Composite retries prove exact replay and content mismatch behavior in Sim, Turso, + PostgreSQL, and supported Redis topology without scanning the parent stream. +- Live local E2E demonstrates delete/null release and reclaim through the server API. +- Formatting, strict Clippy, readability, workspace tests, DST review, independent + exact-GitHub-head review, Greptile, and CI all pass. + +## Consequences + +### Positive + +- Declared-key ownership exactly follows current non-deleted state, including the + authoritative empty set. +- Snapshot-only state survives its first journal write without an ambiguous source + generation or stale-snapshot resurrection. +- Query projections cannot remain silently stale after a source write, repair race, + migration, or delayed delivery. +- Long-lived streams recover through bounded pages without an artificial lifetime + event limit. +- Key, actor, backfill, and query reads share the same source-authority rules. + +### Negative + +- Keyed writes replace the entity's reverse-index rows and take the type/stream + locks required for atomic validation. +- Projection reads may return retryable 503 responses while bounded dirty work drains. +- PostgreSQL adds row-level source/catalog triggers during the compatibility cutover. +- The first deployment performs a full versioned key repair and a one-time projection + seed. +- Deployment requires an old-binary drain because the journal format gains a durable + materialization control record. +- Tenant publication is single-process until a durable distributed generation lease + replaces the in-memory barrier. +- Exact composite-claim convergence is single-process until a durable distributed + claim lease or reservation replaces its bounded in-memory serializer. + +### Risks + +- Publishing ownership before validating every claim could release a valid owner on + a rejected batch. Stores validate the complete transaction before mutation. +- Clearing a dirty marker outside the exact source transaction could certify a stale + row. The fence and clear are one transaction, and unfenced writes re-mark it. +- A partially introduced Turso ledger could permanently skip upgrade seeding. Its + table, index, and seed share one immediate transaction. +- Overlapping an old reader with a materialized stream could lose the retired + baseline during replay. Rollout requires a verified old-reader drain; database + triggers mitigate old writes but do not relax that barrier. +- Treating a control event name alone as internal could drop a legal domain action. + Full payload discrimination is required everywhere, including snapshot retirement. + +### DST Compliance + +- Simulation-visible code uses `sim_now()`/`sim_uuid()` and ordered collections. +- Replay and repair are bounded by explicit budgets and deterministic high-water + coordinates; no thread or ambient I/O is introduced. +- The existing production-only retry sleep and timing metrics retain their scoped + `determinism-ok` annotations. + +## Non-Goals + +- Making Turso declared-key ownership authoritative. +- Changing key hashing or general non-key OData semantics. +- Supporting an old reader after the first materialization record is enabled. +- Merging or deploying this arena PR before adjudication. + +## Alternatives Considered + +1. **Sentinel rows for empty ownership** — rejected because they would participate + in uniqueness and encode invented domain values. +2. **Asynchronous key cleanup** — rejected because reclaim can cross the cleanup and + journal/key ownership can diverge on failure. +3. **Numeric newest-source wins** — rejected because equal-sequence replacement and + stale higher snapshots/catalog rows are valid historical shapes. +4. **SQL-only repair** — rejected because current state and tombstone precedence + require transition-table replay. +5. **In-memory projection invalidation** — rejected because crash and multi-process + delivery gaps require a durable transactionally maintained marker. + +## Rollback Policy + +Before any valid `Temper.Internal.StateMaterialization.v1` record is committed, the +new binary may be drained and the code rolled back; the key index and query projection +are derived data and can be rebuilt. + +After the first materialization record commits, an ordinary rollback to an older +reader is unsafe: the authoritative snapshot may already be retired and the older +binary cannot reconstruct the baseline from the control record. From that point, +recovery is forward-fix only with a reader that understands this record. The only +rollback to an old binary is restoration of a full pre-enable database backup while +all processes are stopped, accepting loss of writes after that backup. No mixed old/new +reader overlap is permitted during either direction of the cutover. diff --git a/test-fixtures/specs/keyed_doc.ioa.toml b/test-fixtures/specs/keyed_doc.ioa.toml index ef8420237..74b17c0ce 100644 --- a/test-fixtures/specs/keyed_doc.ioa.toml +++ b/test-fixtures/specs/keyed_doc.ioa.toml @@ -30,3 +30,11 @@ from = ["New"] to = "Ready" params = ["WorkspaceId", "Path"] hint = "Create the doc with its workspace + path (its declared key)." + +[[action]] +name = "Rekey" +kind = "input" +from = ["Ready"] +to = "Ready" +params = ["WorkspaceId", "Path"] +hint = "Replace the doc's declared-key fields."