diff --git a/crates/temper-store-turso/src/lib.rs b/crates/temper-store-turso/src/lib.rs index 231b54209..efd85c52d 100644 --- a/crates/temper-store-turso/src/lib.rs +++ b/crates/temper-store-turso/src/lib.rs @@ -6,6 +6,7 @@ //! trait from `temper-runtime` using libSQL (Turso-compatible). mod metrics; +mod migrations; mod retry; pub mod router; pub mod schema; diff --git a/crates/temper-store-turso/src/migrations/catalog.rs b/crates/temper-store-turso/src/migrations/catalog.rs new file mode 100644 index 000000000..b027d90bb --- /dev/null +++ b/crates/temper-store-turso/src/migrations/catalog.rs @@ -0,0 +1,459 @@ +use sha2::{Digest, Sha256}; + +use super::ots_rebuild::OTS_REBUILD_DEFINITION; +use crate::schema; + +pub(super) const VALIDATION_MANIFEST_VERSION: &str = "length-prefixed-schema-snapshot-v10"; + +#[derive(Clone, Copy, Debug)] +pub(super) enum MigrationStep { + Sql(&'static str), + AddColumn { + table: &'static str, + column: &'static str, + sql: &'static str, + }, + RebuildOtsTrajectories, +} + +#[derive(Clone, Copy, Debug)] +pub(super) struct Migration { + pub version: u32, + pub name: &'static str, + pub steps: &'static [MigrationStep], +} + +impl Migration { + pub fn checksum(&self, schema_manifest: &str) -> String { + let mut hasher = Sha256::new(); + hash_part(&mut hasher, b"temper-turso-migration-v1"); + hash_part(&mut hasher, &self.version.to_be_bytes()); + hash_part(&mut hasher, self.name.as_bytes()); + hash_part(&mut hasher, VALIDATION_MANIFEST_VERSION.as_bytes()); + for step in self.steps { + match step { + MigrationStep::Sql(sql) => { + hash_part(&mut hasher, b"sql"); + hash_part(&mut hasher, sql.as_bytes()); + } + MigrationStep::AddColumn { table, column, sql } => { + hash_part(&mut hasher, b"add-column"); + hash_part(&mut hasher, table.as_bytes()); + hash_part(&mut hasher, column.as_bytes()); + hash_part(&mut hasher, sql.as_bytes()); + } + MigrationStep::RebuildOtsTrajectories => { + let definition = &OTS_REBUILD_DEFINITION; + hash_part(&mut hasher, b"rebuild-ots-trajectories"); + hash_part(&mut hasher, definition.algorithm_version.as_bytes()); + hash_part(&mut hasher, definition.table.as_bytes()); + hash_part(&mut hasher, definition.temporary_table.as_bytes()); + for column in definition.required_columns { + hash_part(&mut hasher, column.name.as_bytes()); + hash_part(&mut hasher, column.affinity.as_bytes()); + hash_part(&mut hasher, &[u8::from(column.not_null)]); + hash_part(&mut hasher, column.default.unwrap_or("").as_bytes()); + hash_part(&mut hasher, &column.primary_key_position.to_be_bytes()); + } + let column = definition.updated_at_column; + hash_part(&mut hasher, column.name.as_bytes()); + hash_part(&mut hasher, column.affinity.as_bytes()); + hash_part(&mut hasher, &[u8::from(column.not_null)]); + hash_part(&mut hasher, column.default.unwrap_or("").as_bytes()); + hash_part(&mut hasher, &column.primary_key_position.to_be_bytes()); + for sequence in definition.forbidden_table_sql_sequences { + hash_part(&mut hasher, &(sequence.len() as u64).to_be_bytes()); + for token in *sequence { + hash_part(&mut hasher, token.as_bytes()); + } + } + hash_part(&mut hasher, definition.schema_tables_query.as_bytes()); + hash_part(&mut hasher, definition.dependent_objects_query.as_bytes()); + hash_part(&mut hasher, definition.create_temporary_sql.as_bytes()); + hash_part(&mut hasher, definition.copy_sql.as_bytes()); + hash_part(&mut hasher, definition.drop_sql.as_bytes()); + hash_part(&mut hasher, definition.rename_sql.as_bytes()); + } + } + } + hash_part(&mut hasher, schema_manifest.as_bytes()); + format!("{:x}", hasher.finalize()) + } +} + +fn hash_part(hasher: &mut Sha256, bytes: &[u8]) { + hasher.update((bytes.len() as u64).to_be_bytes()); + hasher.update(bytes); +} + +const JOURNAL_STEPS: &[MigrationStep] = &[ + MigrationStep::Sql(schema::CREATE_EVENTS_TABLE), + MigrationStep::AddColumn { + table: "events", + column: "segment_index", + sql: schema::ALTER_EVENTS_ADD_SEGMENT_INDEX, + }, + MigrationStep::Sql(schema::CREATE_EVENTS_ENTITY_INDEX), + MigrationStep::Sql(schema::CREATE_EVENT_SEGMENTS_TABLE), + MigrationStep::Sql(schema::CREATE_EVENT_SEGMENTS_OPEN_INDEX), + MigrationStep::Sql(schema::CREATE_SNAPSHOTS_TABLE), + MigrationStep::Sql(schema::CREATE_SNAPSHOT_HISTORY_TABLE), + MigrationStep::Sql(schema::CREATE_SNAPSHOT_HISTORY_ENTITY_INDEX), +]; + +const SPEC_INTEGRATION_STEPS: &[MigrationStep] = &[ + MigrationStep::Sql(schema::CREATE_SPECS_TABLE), + MigrationStep::AddColumn { + table: "specs", + column: "content_hash", + sql: schema::ALTER_SPECS_ADD_CONTENT_HASH, + }, + MigrationStep::AddColumn { + table: "specs", + column: "committed", + sql: schema::ALTER_SPECS_ADD_COMMITTED, + }, + MigrationStep::Sql(schema::CREATE_TENANT_CONSTRAINTS_TABLE), + MigrationStep::Sql(schema::CREATE_WASM_MODULES_TABLE), + MigrationStep::AddColumn { + table: "wasm_modules", + column: "source", + sql: schema::ADD_WASM_MODULES_SOURCE_COLUMN, + }, + MigrationStep::Sql(schema::CREATE_WASM_INVOCATION_LOGS_TABLE), + MigrationStep::Sql(schema::CREATE_WASM_INVOCATION_LOGS_TENANT_INDEX), + MigrationStep::Sql(schema::CREATE_WASM_INVOCATION_LOGS_MODULE_INDEX), + MigrationStep::Sql(schema::CREATE_WASM_INVOCATION_LOGS_CREATED_INDEX), +]; + +const AUTHORIZATION_STEPS: &[MigrationStep] = &[ + MigrationStep::Sql(schema::CREATE_PENDING_DECISIONS_TABLE), + MigrationStep::Sql(schema::CREATE_PENDING_DECISIONS_TENANT_INDEX), + MigrationStep::Sql(schema::CREATE_PENDING_DECISIONS_STATUS_INDEX), + MigrationStep::Sql(schema::CREATE_TENANT_POLICIES_TABLE), + MigrationStep::Sql(schema::CREATE_POLICIES_TABLE), + MigrationStep::AddColumn { + table: "policies", + column: "enabled", + sql: schema::ALTER_POLICIES_ADD_ENABLED, + }, + MigrationStep::Sql(schema::CREATE_POLICY_DENIAL_PATTERNS_TABLE), + MigrationStep::Sql(schema::CREATE_POLICY_DENIAL_PATTERNS_TENANT_INDEX), + MigrationStep::Sql(schema::CREATE_PUBLISHED_ARTIFACTS_TABLE), + MigrationStep::Sql(schema::CREATE_PUBLISHED_ARTIFACTS_OWNER_INDEX), + MigrationStep::Sql(schema::CREATE_PUBLISHED_ARTIFACTS_SOURCE_INDEX), +]; + +const APP_PLATFORM_STEPS: &[MigrationStep] = &[ + MigrationStep::Sql(schema::CREATE_TENANT_INSTALLED_APPS_TABLE), + MigrationStep::AddColumn { + table: "tenant_installed_apps", + column: "app_version", + sql: schema::ALTER_INSTALLED_APPS_ADD_APP_VERSION, + }, + MigrationStep::AddColumn { + table: "tenant_installed_apps", + column: "source_kind", + sql: schema::ALTER_INSTALLED_APPS_ADD_SOURCE_KIND, + }, + MigrationStep::AddColumn { + table: "tenant_installed_apps", + column: "app_ref", + sql: schema::ALTER_INSTALLED_APPS_ADD_APP_REF, + }, + MigrationStep::AddColumn { + table: "tenant_installed_apps", + column: "version_hash", + sql: schema::ALTER_INSTALLED_APPS_ADD_VERSION_HASH, + }, + MigrationStep::AddColumn { + table: "tenant_installed_apps", + column: "pinned_version_hash", + sql: schema::ALTER_INSTALLED_APPS_ADD_PINNED_VERSION_HASH, + }, + MigrationStep::AddColumn { + table: "tenant_installed_apps", + column: "current_version_hash", + sql: schema::ALTER_INSTALLED_APPS_ADD_CURRENT_VERSION_HASH, + }, + MigrationStep::AddColumn { + table: "tenant_installed_apps", + column: "follow_policy", + sql: schema::ALTER_INSTALLED_APPS_ADD_FOLLOW_POLICY, + }, + MigrationStep::AddColumn { + table: "tenant_installed_apps", + column: "closure_id", + sql: schema::ALTER_INSTALLED_APPS_ADD_CLOSURE_ID, + }, + MigrationStep::AddColumn { + table: "tenant_installed_apps", + column: "registry_url", + sql: schema::ALTER_INSTALLED_APPS_ADD_REGISTRY_URL, + }, + MigrationStep::AddColumn { + table: "tenant_installed_apps", + column: "registry_tenant", + sql: schema::ALTER_INSTALLED_APPS_ADD_REGISTRY_TENANT, + }, + MigrationStep::AddColumn { + table: "tenant_installed_apps", + column: "bundle_digest", + sql: schema::ALTER_INSTALLED_APPS_ADD_BUNDLE_DIGEST, + }, + MigrationStep::AddColumn { + table: "tenant_installed_apps", + column: "spec_digest", + sql: schema::ALTER_INSTALLED_APPS_ADD_SPEC_DIGEST, + }, + MigrationStep::AddColumn { + table: "tenant_installed_apps", + column: "policy_digest", + sql: schema::ALTER_INSTALLED_APPS_ADD_POLICY_DIGEST, + }, + MigrationStep::AddColumn { + table: "tenant_installed_apps", + column: "wasm_digest", + sql: schema::ALTER_INSTALLED_APPS_ADD_WASM_DIGEST, + }, + MigrationStep::AddColumn { + table: "tenant_installed_apps", + column: "content_digest", + sql: schema::ALTER_INSTALLED_APPS_ADD_CONTENT_DIGEST, + }, + MigrationStep::AddColumn { + table: "tenant_installed_apps", + column: "seed_digest", + sql: schema::ALTER_INSTALLED_APPS_ADD_SEED_DIGEST, + }, + MigrationStep::AddColumn { + table: "tenant_installed_apps", + column: "last_reconciled_at", + sql: schema::ALTER_INSTALLED_APPS_ADD_LAST_RECONCILED_AT, + }, + MigrationStep::AddColumn { + table: "tenant_installed_apps", + column: "status", + sql: schema::ALTER_INSTALLED_APPS_ADD_STATUS, + }, + MigrationStep::Sql(schema::CREATE_TENANT_REGISTRY_TABLE), + MigrationStep::Sql(schema::CREATE_TENANT_USERS_TABLE), + MigrationStep::Sql(schema::CREATE_TENANT_USERS_USER_INDEX), + MigrationStep::Sql(schema::CREATE_TENANT_SECRETS_TABLE), +]; + +const TRAJECTORY_EVOLUTION_STEPS: &[MigrationStep] = &[ + MigrationStep::Sql(schema::CREATE_TRAJECTORIES_TABLE), + MigrationStep::AddColumn { + table: "trajectories", + column: "agent_id", + sql: schema::ALTER_TRAJECTORIES_ADD_AGENT_ID, + }, + MigrationStep::AddColumn { + table: "trajectories", + column: "session_id", + sql: schema::ALTER_TRAJECTORIES_ADD_SESSION_ID, + }, + MigrationStep::AddColumn { + table: "trajectories", + column: "authz_denied", + sql: schema::ALTER_TRAJECTORIES_ADD_AUTHZ_DENIED, + }, + MigrationStep::AddColumn { + table: "trajectories", + column: "denied_resource", + sql: schema::ALTER_TRAJECTORIES_ADD_DENIED_RESOURCE, + }, + MigrationStep::AddColumn { + table: "trajectories", + column: "denied_module", + sql: schema::ALTER_TRAJECTORIES_ADD_DENIED_MODULE, + }, + MigrationStep::AddColumn { + table: "trajectories", + column: "source", + sql: schema::ALTER_TRAJECTORIES_ADD_SOURCE, + }, + MigrationStep::AddColumn { + table: "trajectories", + column: "spec_governed", + sql: schema::ALTER_TRAJECTORIES_ADD_SPEC_GOVERNED, + }, + MigrationStep::AddColumn { + table: "trajectories", + column: "request_body", + sql: schema::ALTER_TRAJECTORIES_ADD_REQUEST_BODY, + }, + MigrationStep::AddColumn { + table: "trajectories", + column: "intent", + sql: schema::ALTER_TRAJECTORIES_ADD_INTENT, + }, + MigrationStep::AddColumn { + table: "trajectories", + column: "matched_policy_ids", + sql: schema::ALTER_TRAJECTORIES_ADD_MATCHED_POLICY_IDS, + }, + MigrationStep::Sql(schema::CREATE_TRAJECTORIES_SUCCESS_INDEX), + MigrationStep::Sql(schema::CREATE_TRAJECTORIES_ENTITY_ACTION_INDEX), + MigrationStep::Sql(schema::CREATE_TRAJECTORIES_AGENT_INDEX), + MigrationStep::Sql(schema::CREATE_FEATURE_REQUESTS_TABLE), + MigrationStep::Sql(schema::CREATE_EVOLUTION_RECORDS_TABLE), + MigrationStep::Sql(schema::CREATE_EVOLUTION_RECORDS_TYPE_INDEX), + MigrationStep::Sql(schema::CREATE_EVOLUTION_RECORDS_STATUS_INDEX), + MigrationStep::Sql(schema::CREATE_DESIGN_TIME_EVENTS_TABLE), + MigrationStep::Sql(schema::CREATE_DESIGN_TIME_EVENTS_TENANT_INDEX), + MigrationStep::Sql(schema::CREATE_OTS_TRAJECTORIES_TABLE), + MigrationStep::AddColumn { + table: "ots_trajectories", + column: "persistence_status", + sql: schema::ALTER_OTS_TRAJECTORIES_ADD_PERSISTENCE_STATUS, + }, + MigrationStep::AddColumn { + table: "ots_trajectories", + column: "persist_attempts", + sql: schema::ALTER_OTS_TRAJECTORIES_ADD_PERSIST_ATTEMPTS, + }, + MigrationStep::AddColumn { + table: "ots_trajectories", + column: "last_error", + sql: schema::ALTER_OTS_TRAJECTORIES_ADD_LAST_ERROR, + }, + MigrationStep::RebuildOtsTrajectories, + MigrationStep::Sql(schema::CREATE_OTS_TRAJECTORIES_AGENT_INDEX), + MigrationStep::Sql(schema::CREATE_OTS_TRAJECTORIES_TENANT_INDEX), + MigrationStep::Sql(schema::CREATE_OTS_TRAJECTORIES_OUTCOME_INDEX), + MigrationStep::Sql(schema::CREATE_OTS_TRAJECTORIES_STATUS_INDEX), +]; + +const QUERY_PLANE_STEPS: &[MigrationStep] = &[ + MigrationStep::Sql(schema::CREATE_BLOBS_TABLE), + MigrationStep::AddColumn { + table: "blobs", + column: "expires_at", + sql: schema::ALTER_BLOBS_ADD_EXPIRES_AT, + }, + MigrationStep::Sql(schema::CREATE_BLOBS_EXPIRES_AT_INDEX), + MigrationStep::Sql(schema::CREATE_ENTITY_CATALOG_TABLE), + MigrationStep::AddColumn { + table: "entity_catalog", + column: "projection_hash", + sql: schema::ALTER_ENTITY_CATALOG_ADD_PROJECTION_HASH, + }, + MigrationStep::AddColumn { + table: "entity_catalog", + column: "fields", + sql: schema::ALTER_ENTITY_CATALOG_ADD_FIELDS, + }, + MigrationStep::AddColumn { + table: "entity_catalog", + column: "state", + sql: schema::ALTER_ENTITY_CATALOG_ADD_STATE, + }, + MigrationStep::Sql(schema::CREATE_ENTITY_CATALOG_TYPE_INDEX), + MigrationStep::Sql(schema::CREATE_ENTITY_CATALOG_STATUS_INDEX), + MigrationStep::Sql(schema::CREATE_ENTITY_FIELD_INDEX_TABLE), + MigrationStep::Sql(schema::CREATE_ENTITY_FIELD_INDEX_LOOKUP), + MigrationStep::Sql(schema::CREATE_ENTITY_FIELD_INDEX_STATUS), +]; + +const DECLARED_INDEX_STEPS: &[MigrationStep] = &[ + MigrationStep::Sql(schema::CREATE_ENTITY_KEY_INDEX_TABLE), + MigrationStep::Sql(schema::CREATE_ENTITY_KEY_INDEX_ENTITY), + MigrationStep::Sql(schema::CREATE_ENTITY_VECTOR_INDEX_TABLE), + MigrationStep::Sql(schema::CREATE_ENTITY_VECTOR_INDEX_PARTITION), + MigrationStep::Sql(schema::CREATE_ENTITY_VECTOR_INDEX_ENTITY), + MigrationStep::Sql(schema::CREATE_VECTOR_INDEX_BACKFILL_WATERMARK), +]; + +pub(super) const MIGRATIONS: &[Migration] = &[ + Migration { + version: 1, + name: "event-journal-and-snapshots", + steps: JOURNAL_STEPS, + }, + Migration { + version: 2, + name: "specs-constraints-and-integrations", + steps: SPEC_INTEGRATION_STEPS, + }, + Migration { + version: 3, + name: "authorization-and-artifacts", + steps: AUTHORIZATION_STEPS, + }, + Migration { + version: 4, + name: "apps-platform-and-secrets", + steps: APP_PLATFORM_STEPS, + }, + Migration { + version: 5, + name: "trajectories-and-evolution", + steps: TRAJECTORY_EVOLUTION_STEPS, + }, + Migration { + version: 6, + name: "blob-and-query-plane", + steps: QUERY_PLANE_STEPS, + }, + Migration { + version: 7, + name: "declared-key-and-vector-indexes", + steps: DECLARED_INDEX_STEPS, + }, +]; + +#[cfg(test)] +mod tests { + use super::super::runner::expected_checksums; + use super::super::schema_verify::EXTRA_INDEX_POLICY; + + const PRE_OWNER_AWARE_CHECKSUMS: &[&str] = &[ + "78bafc020d87a65741a6f7c117604f693d5eb265d75b178db1737f8934da8069", + "83bc0de0ecf597a24ebe14fc6636b9b70b3cc76b6342b326afb583715e5d18b9", + "54a077e4353c6df79dce2029cded8ce148c50be90400c4893dea21752adde4ea", + "6dfcf2905113a7943f80c44da094cb5b53b35633298b1a8fdf933df127b1ee8d", + "f63408461791d04d70082f996c5f7bd620d3f6af505b9c98ffa7d3a63df38d75", + "5347da7626a3ca311ba8295e46fc7a0a22f0f6eb1944f09aee85dabca3a7fc4d", + "a8b51d91118d03697d98db8a3ff55fbed5967a71e7305dcd13876a56ad206a7c", + ]; + + const RELEASED_CHECKSUMS: &[&str] = &[ + "aa159f54e46819312662448552d6ebdd56e5fe0e31d4f7619b28c3c9272521d2", + "5ee4fcfe1f9ff6a7b0d1d4c4081a7ec375491abc0f5ba96ec42d1eb405f137cb", + "1634a9cdd48acff70f20d2ef78a6b91285a016bd53dbd459b7e9c663650f691c", + "e25ad7da3742dc84c0a2ef1b0713b1858b82af040fd503f5cfd3aed4e7151bef", + "5606020fefa283e048c7a72ae8bd7db3dc91336a946e8305ac6993e606f50aeb", + "78aa56edc91e690e39ffeacc46c85804b3034d2368a480eae1769dfaa516b111", + "1e4e7af4c85e857bf0a55890d4c4142f95cc5053a252c70309794599c481603b", + ]; + + #[tokio::test] + async fn released_migration_checksums_are_stable() { + let checksums = expected_checksums().await.expect("expected checksums"); + assert!(checksums.len() >= RELEASED_CHECKSUMS.len()); + assert_eq!(&checksums[..RELEASED_CHECKSUMS.len()], RELEASED_CHECKSUMS); + } + + #[tokio::test] + async fn owner_aware_index_inventory_changes_durable_checksums() { + assert!( + EXTRA_INDEX_POLICY.contains("sqlite-identifier-owners-v2"), + "owner-aware index inventory must have an explicit durable policy version: {EXTRA_INDEX_POLICY}" + ); + + let checksums = expected_checksums().await.expect("expected checksums"); + assert!(checksums.len() >= PRE_OWNER_AWARE_CHECKSUMS.len()); + for (index, (actual, previous)) in + checksums.iter().zip(PRE_OWNER_AWARE_CHECKSUMS).enumerate() + { + assert_ne!( + actual, + previous, + "migration version {} reused its pre-owner-aware durable checksum", + index + 1 + ); + } + } +} diff --git a/crates/temper-store-turso/src/migrations/ledger.rs b/crates/temper-store-turso/src/migrations/ledger.rs new file mode 100644 index 000000000..6836a802b --- /dev/null +++ b/crates/temper-store-turso/src/migrations/ledger.rs @@ -0,0 +1,82 @@ +use libsql::Connection; +use temper_runtime::persistence::PersistenceError; + +use super::schema_sql::normalize_schema_ddl; + +pub(super) const CREATE_MIGRATION_LEDGER: &str = "\ +CREATE TABLE IF NOT EXISTS temper_schema_migrations ( + version INTEGER PRIMARY KEY CHECK (version > 0), + name TEXT NOT NULL UNIQUE, + checksum TEXT NOT NULL CHECK (length(checksum) = 64), + applied_at TEXT NOT NULL DEFAULT (datetime('now')) +);"; + +pub(super) async fn validate_ledger_schema( + connection: &Connection, +) -> Result<(), PersistenceError> { + let mut rows = connection + .query( + "SELECT type, sql FROM sqlite_schema + WHERE name = 'temper_schema_migrations' ORDER BY type LIMIT 1", + (), + ) + .await + .map_err(|error| ledger_error("inspect migration-ledger schema", error))?; + let row = rows + .next() + .await + .map_err(|error| ledger_error("read migration-ledger schema", error))? + .ok_or_else(|| { + PersistenceError::Storage("Turso migration ledger table is missing".to_string()) + })?; + let kind = row + .get::(0) + .map_err(|error| ledger_error("decode migration-ledger object kind", error))?; + if kind != "table" { + return Err(PersistenceError::Storage(format!( + "Turso migration ledger capability must be a table, found {kind}" + ))); + } + let actual = row + .get::(1) + .map_err(|error| ledger_error("decode migration-ledger schema", error))?; + drop(rows); + + if normalize_schema_ddl(&actual) != normalize_schema_ddl(CREATE_MIGRATION_LEDGER) { + return Err(PersistenceError::Storage(format!( + "Turso migration ledger has incompatible schema: expected {}, found {}", + normalize_schema_ddl(CREATE_MIGRATION_LEDGER), + normalize_schema_ddl(&actual) + ))); + } + + let mut triggers = connection + .query( + "SELECT name FROM sqlite_schema + WHERE type = 'trigger' + AND tbl_name COLLATE NOCASE = 'temper_schema_migrations' + ORDER BY name LIMIT 1", + (), + ) + .await + .map_err(|error| ledger_error("inspect migration-ledger triggers", error))?; + if let Some(row) = triggers + .next() + .await + .map_err(|error| ledger_error("read migration-ledger trigger", error))? + { + let name = row + .get::(0) + .map_err(|error| ledger_error("decode migration-ledger trigger", error))?; + return Err(PersistenceError::Storage(format!( + "Turso migration ledger has unsupported trigger '{name}'" + ))); + } + Ok(()) +} + +fn ledger_error(context: &str, error: libsql::Error) -> PersistenceError { + PersistenceError::Storage(format!( + "Turso schema migration failed while attempting to {context}: {error} ({error:?})" + )) +} diff --git a/crates/temper-store-turso/src/migrations/ledger_tests.rs b/crates/temper-store-turso/src/migrations/ledger_tests.rs new file mode 100644 index 000000000..0526aa97d --- /dev/null +++ b/crates/temper-store-turso/src/migrations/ledger_tests.rs @@ -0,0 +1,183 @@ +use libsql::Builder; + +use super::catalog::MIGRATIONS; +use super::runner::migrate; + +#[tokio::test] +async fn compact_equivalent_ledger_definition_is_accepted() { + let directory = tempfile::tempdir().expect("temporary database directory"); + let database = Builder::new_local(directory.path().join("compact-ledger.db")) + .build() + .await + .expect("build compact-ledger database"); + let connection = database.connect().expect("connect compact-ledger database"); + connection + .execute( + "CREATE TABLE temper_schema_migrations(version INTEGER PRIMARY KEY CHECK(version>0),name TEXT NOT NULL UNIQUE,checksum TEXT NOT NULL CHECK(length(checksum)=64),applied_at TEXT NOT NULL DEFAULT(datetime('now')))", + (), + ) + .await + .expect("create semantically identical compact ledger"); + + migrate(&connection) + .await + .expect("formatting must not make an identical ledger incompatible"); + + let mut rows = connection + .query("SELECT COUNT(*) FROM temper_schema_migrations", ()) + .await + .expect("query ledger count"); + let count = rows + .next() + .await + .expect("read ledger count") + .expect("ledger count row") + .get::(0) + .expect("decode ledger count"); + assert_eq!(count, MIGRATIONS.len() as i64); +} + +#[tokio::test] +async fn merged_constraint_tokens_do_not_satisfy_ledger_contract() { + let directory = tempfile::tempdir().expect("temporary database directory"); + let database = Builder::new_local(directory.path().join("weak-ledger.db")) + .build() + .await + .expect("build weak-ledger database"); + let connection = database.connect().expect("connect weak-ledger database"); + connection + .execute( + "CREATE TABLE temper_schema_migrations( + version INTEGER PRIMARYKEY CHECK(version>0), + name TEXTNOT NULL UNIQUE, + checksum TEXTNOT NULL CHECK(length(checksum)=64), + applied_at TEXTNOT NULL DEFAULT(datetime('now')) + )", + (), + ) + .await + .expect("create structurally weaker ledger"); + + let error = migrate(&connection) + .await + .expect_err("merged SQL words must not satisfy separate constraint tokens"); + assert!(error.to_string().contains("incompatible schema"), "{error}"); + let mut columns = connection + .query("PRAGMA table_xinfo(temper_schema_migrations)", ()) + .await + .expect("inspect weak ledger columns"); + while let Some(row) = columns.next().await.expect("read weak ledger column") { + assert_eq!(row.get::(3).expect("not-null flag"), 0); + assert_eq!(row.get::(5).expect("primary-key position"), 0); + } +} + +#[tokio::test] +async fn ignored_ledger_insert_prevents_schema_commit_and_readiness() { + let directory = tempfile::tempdir().expect("temporary database directory"); + let database = Builder::new_local(directory.path().join("ignored-ledger-insert.db")) + .build() + .await + .expect("build ignored-ledger-insert database"); + let connection = database + .connect() + .expect("connect ignored-ledger-insert database"); + connection + .execute( + "CREATE TABLE temper_schema_migrations( + version INTEGER PRIMARY KEY CHECK(version>0), + name TEXT NOT NULL UNIQUE, + checksum TEXT NOT NULL CHECK(length(checksum)=64), + applied_at TEXT NOT NULL DEFAULT(datetime('now')) + )", + (), + ) + .await + .expect("create exact ledger"); + connection + .execute( + "CREATE TRIGGER ignore_migration_ledger_insert + BEFORE INSERT ON temper_schema_migrations + BEGIN SELECT RAISE(IGNORE); END", + (), + ) + .await + .expect("create ledger insert trigger"); + + let error = migrate(&connection) + .await + .expect_err("a migration without a retained ledger row must not reach readiness"); + assert!(error.to_string().contains("migration ledger"), "{error}"); + assert_eq!( + scalar_i64(&connection, "SELECT COUNT(*) FROM temper_schema_migrations").await, + 0 + ); + assert_eq!( + scalar_i64( + &connection, + "SELECT COUNT(*) FROM sqlite_schema WHERE type = 'table' AND name = 'events'" + ) + .await, + 0 + ); +} + +#[tokio::test] +async fn differently_cased_ledger_trigger_owner_is_rejected_before_migration() { + let directory = tempfile::tempdir().expect("temporary database directory"); + let database = Builder::new_local(directory.path().join("case-folded-ledger-trigger.db")) + .build() + .await + .expect("build case-folded-ledger-trigger database"); + let connection = database + .connect() + .expect("connect case-folded-ledger-trigger database"); + connection + .execute(super::ledger::CREATE_MIGRATION_LEDGER, ()) + .await + .expect("create exact migration ledger"); + connection + .execute( + "CREATE TRIGGER case_folded_ledger_audit + AFTER INSERT ON TEMPER_SCHEMA_MIGRATIONS + BEGIN SELECT 1; END", + (), + ) + .await + .expect("create benign trigger with differently cased ledger owner"); + + let error = migrate(&connection) + .await + .expect_err("every trigger on the migration ledger must be rejected"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains("unsupported trigger"), "{diagnostic}"); + assert!( + diagnostic.contains("case_folded_ledger_audit"), + "{diagnostic}" + ); + assert_eq!( + scalar_i64(&connection, "SELECT COUNT(*) FROM temper_schema_migrations").await, + 0 + ); + assert_eq!( + scalar_i64( + &connection, + "SELECT COUNT(*) FROM sqlite_schema WHERE type = 'table' AND name = 'events'" + ) + .await, + 0 + ); +} + +async fn scalar_i64(connection: &libsql::Connection, sql: &str) -> i64 { + let mut rows = connection + .query(sql, ()) + .await + .expect("query integer scalar"); + rows.next() + .await + .expect("read integer scalar") + .expect("integer scalar row") + .get::(0) + .expect("decode integer scalar") +} diff --git a/crates/temper-store-turso/src/migrations/mod.rs b/crates/temper-store-turso/src/migrations/mod.rs new file mode 100644 index 000000000..c26f780f2 --- /dev/null +++ b/crates/temper-store-turso/src/migrations/mod.rs @@ -0,0 +1,26 @@ +mod catalog; +mod ledger; +#[cfg(test)] +mod ledger_tests; +#[cfg(test)] +mod ots_current_tests; +mod ots_rebuild; +#[cfg(test)] +mod ots_rebuild_tests; +#[cfg(test)] +mod remote_probe_tests; +mod runner; +mod schema_manifest; +mod schema_ots_probe; +mod schema_ots_trigger; +mod schema_snapshot; +mod schema_sql; +mod schema_trigger; +mod schema_verify; +#[cfg(test)] +mod schema_verify_tests; + +pub(crate) use runner::migrate; + +#[cfg(test)] +mod tests; diff --git a/crates/temper-store-turso/src/migrations/ots_current_tests.rs b/crates/temper-store-turso/src/migrations/ots_current_tests.rs new file mode 100644 index 000000000..188cb64f3 --- /dev/null +++ b/crates/temper-store-turso/src/migrations/ots_current_tests.rs @@ -0,0 +1,152 @@ +use libsql::{Builder, Connection, params}; + +use super::catalog::MIGRATIONS; +use super::runner::migrate; +use crate::store::ots::PERSIST_OTS_TRAJECTORY_SQL; + +#[tokio::test] +async fn current_ots_shape_preserves_harmless_inbound_reference() { + let (_directory, connection) = current_ots_with_child("current-inbound", "NO ACTION").await; + + migrate(&connection) + .await + .expect("an already-current table needs no destructive inbound-FK gate"); + + assert_eq!( + scalar_i64(&connection, "SELECT COUNT(*) FROM ots_trajectories").await, + 1 + ); + assert_eq!( + scalar_i64(&connection, "SELECT COUNT(*) FROM current_ots_child").await, + 1 + ); + assert_eq!( + scalar_i64(&connection, "SELECT COUNT(*) FROM temper_schema_migrations").await, + MIGRATIONS.len() as i64 + ); +} + +#[tokio::test] +async fn current_ots_cascade_child_survives_existing_id_production_persist() { + assert_existing_id_persist_preserves_child("current-cascade", "CASCADE").await; +} + +#[tokio::test] +async fn current_ots_restrict_child_allows_existing_id_production_persist() { + assert_existing_id_persist_preserves_child("current-restrict", "RESTRICT").await; +} + +async fn assert_existing_id_persist_preserves_child(label: &str, on_delete: &str) { + let (_directory, connection) = current_ots_with_child(label, on_delete).await; + + migrate(&connection) + .await + .expect("current OTS schema with an inbound reference remains compatible"); + connection + .execute( + PERSIST_OTS_TRAJECTORY_SQL, + params![ + "trajectory-current".to_string(), + "tenant-updated".to_string(), + "agent-updated".to_string(), + "session-updated".to_string(), + "persisted-updated".to_string(), + 7_i64, + "{\"stage\":\"updated\"}".to_string(), + ], + ) + .await + .expect("existing-ID production persist must not delete or reject inbound references"); + + assert_eq!( + scalar_i64(&connection, "SELECT COUNT(*) FROM current_ots_child").await, + 1, + "ON DELETE {on_delete} child must survive an existing-ID production persist" + ); + assert_eq!( + scalar_i64( + &connection, + "SELECT turn_count FROM ots_trajectories + WHERE trajectory_id = 'trajectory-current'" + ) + .await, + 7 + ); +} + +async fn current_ots_with_child(label: &str, on_delete: &str) -> (tempfile::TempDir, Connection) { + let directory = tempfile::tempdir().expect("temporary database directory"); + let database = Builder::new_local(directory.path().join(format!("{label}.db"))) + .build() + .await + .expect("build current OTS database"); + let connection = database.connect().expect("connect current OTS database"); + connection + .execute("PRAGMA foreign_keys = ON", ()) + .await + .expect("enable foreign keys"); + connection + .execute( + "CREATE TABLE ots_trajectories ( + trajectory_id TEXT PRIMARY KEY, + tenant TEXT NOT NULL, + agent_id TEXT NOT NULL, + session_id TEXT, + outcome TEXT NOT NULL DEFAULT 'unknown', + entity_type TEXT, + turn_count INTEGER NOT NULL DEFAULT 0, + data TEXT NOT NULL, + persistence_status TEXT NOT NULL DEFAULT 'persisted', + persist_attempts INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + )", + (), + ) + .await + .expect("create current OTS table"); + connection + .execute( + &format!( + "CREATE TABLE current_ots_child ( + id TEXT PRIMARY KEY, + trajectory_id TEXT NOT NULL + REFERENCES ots_trajectories(trajectory_id) ON DELETE {on_delete} + )" + ), + (), + ) + .await + .expect("create current OTS child table"); + connection + .execute( + "INSERT INTO ots_trajectories (trajectory_id, tenant, agent_id, data) + VALUES ('trajectory-current', 'tenant-a', 'agent-a', '{}')", + (), + ) + .await + .expect("insert current OTS parent"); + connection + .execute( + "INSERT INTO current_ots_child (id, trajectory_id) + VALUES ('child-current', 'trajectory-current')", + (), + ) + .await + .expect("insert current OTS child"); + (directory, connection) +} + +async fn scalar_i64(connection: &libsql::Connection, sql: &str) -> i64 { + let mut rows = connection + .query(sql, ()) + .await + .expect("query integer scalar"); + rows.next() + .await + .expect("read integer scalar") + .expect("integer scalar row") + .get::(0) + .expect("decode integer scalar") +} diff --git a/crates/temper-store-turso/src/migrations/ots_rebuild.rs b/crates/temper-store-turso/src/migrations/ots_rebuild.rs new file mode 100644 index 000000000..59eb5affb --- /dev/null +++ b/crates/temper-store-turso/src/migrations/ots_rebuild.rs @@ -0,0 +1,497 @@ +use std::collections::BTreeMap; + +use libsql::Connection; +use temper_runtime::persistence::PersistenceError; + +use super::catalog::Migration; +use super::runner::{execute_step, schema_object_kind}; +use super::schema_snapshot::{IndexColumn, index_columns, normalize_default, type_affinity}; +use super::schema_sql::contains_sequence; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(super) struct OtsColumnDefinition { + pub name: &'static str, + pub affinity: &'static str, + pub not_null: bool, + pub default: Option<&'static str>, + pub primary_key_position: i64, +} + +#[derive(Clone, Copy, Debug)] +pub(super) struct OtsRebuildDefinition { + pub algorithm_version: &'static str, + pub table: &'static str, + pub temporary_table: &'static str, + pub required_columns: &'static [OtsColumnDefinition], + pub updated_at_column: OtsColumnDefinition, + pub forbidden_table_sql_sequences: &'static [&'static [&'static str]], + pub schema_tables_query: &'static str, + pub dependent_objects_query: &'static str, + pub create_temporary_sql: &'static str, + pub copy_sql: &'static str, + pub drop_sql: &'static str, + pub rename_sql: &'static str, +} + +const REQUIRED_COLUMNS: &[OtsColumnDefinition] = &[ + column("trajectory_id", "TEXT", false, None, 1), + column("tenant", "TEXT", true, None, 0), + column("agent_id", "TEXT", true, None, 0), + column("session_id", "TEXT", false, None, 0), + column("outcome", "TEXT", true, Some("'unknown'"), 0), + column("entity_type", "TEXT", false, None, 0), + column("turn_count", "INTEGER", true, Some("0"), 0), + column("data", "TEXT", true, None, 0), + column("persistence_status", "TEXT", true, Some("'persisted'"), 0), + column("persist_attempts", "INTEGER", true, Some("0"), 0), + column("last_error", "TEXT", false, None, 0), + column("created_at", "TEXT", true, Some("datetime('now')"), 0), +]; + +const UPDATED_AT_COLUMN: OtsColumnDefinition = + column("updated_at", "TEXT", true, Some("datetime('now')"), 0); + +const fn column( + name: &'static str, + affinity: &'static str, + not_null: bool, + default: Option<&'static str>, + primary_key_position: i64, +) -> OtsColumnDefinition { + OtsColumnDefinition { + name, + affinity, + not_null, + default, + primary_key_position, + } +} + +pub(super) const OTS_REBUILD_DEFINITION: OtsRebuildDefinition = OtsRebuildDefinition { + algorithm_version: "preserve-dependent-schema-v8-sqlite-identifier-owners", + table: "ots_trajectories", + temporary_table: "__temper_migration_ots_trajectories", + required_columns: REQUIRED_COLUMNS, + updated_at_column: UPDATED_AT_COLUMN, + forbidden_table_sql_sequences: &[ + &["CHECK"], + &["COLLATE"], + &["GENERATED"], + &["ON", "CONFLICT"], + &["AUTOINCREMENT"], + &["STRICT"], + &["WITHOUT", "ROWID"], + ], + schema_tables_query: "SELECT name FROM sqlite_schema + WHERE type = 'table' AND name NOT GLOB 'sqlite_*' ORDER BY name", + dependent_objects_query: "SELECT type, name, sql FROM sqlite_schema + WHERE tbl_name COLLATE NOCASE = ?1 + AND type IN ('index', 'trigger') AND sql IS NOT NULL + ORDER BY type, name", + create_temporary_sql: "CREATE TABLE __temper_migration_ots_trajectories ( + trajectory_id TEXT PRIMARY KEY, + tenant TEXT NOT NULL, + agent_id TEXT NOT NULL, + session_id TEXT, + outcome TEXT NOT NULL DEFAULT 'unknown', + entity_type TEXT, + turn_count INTEGER NOT NULL DEFAULT 0, + data TEXT NOT NULL, + persistence_status TEXT NOT NULL DEFAULT 'persisted', + persist_attempts INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + )", + copy_sql: "INSERT INTO __temper_migration_ots_trajectories ( + trajectory_id, tenant, agent_id, session_id, outcome, entity_type, + turn_count, data, persistence_status, persist_attempts, last_error, + created_at, updated_at + ) SELECT trajectory_id, tenant, agent_id, session_id, outcome, entity_type, + turn_count, data, persistence_status, persist_attempts, last_error, + created_at, COALESCE(created_at, datetime('now')) + FROM ots_trajectories", + drop_sql: "DROP TABLE ots_trajectories", + rename_sql: "ALTER TABLE __temper_migration_ots_trajectories RENAME TO ots_trajectories", +}; + +#[derive(Debug, Eq, PartialEq)] +struct ObservedColumn { + affinity: String, + not_null: bool, + default: Option, + primary_key_position: i64, + hidden: i64, +} + +#[derive(Debug)] +struct DependentObject { + kind: String, + name: String, + sql: String, +} + +pub(super) async fn rebuild_ots_trajectories( + connection: &Connection, + migration: &Migration, + step_index: usize, +) -> Result<(), PersistenceError> { + let definition = &OTS_REBUILD_DEFINITION; + let columns = table_columns(connection, migration, definition.table).await?; + let already_updated = columns.contains_key(definition.updated_at_column.name); + validate_columns(migration, definition, &columns, already_updated)?; + validate_no_table_constraints(connection, migration, definition, !already_updated).await?; + if already_updated { + return Ok(()); + } + let dependent_objects = dependent_objects(connection, migration, definition).await?; + + if schema_object_kind(connection, definition.temporary_table) + .await? + .is_some() + { + return Err(compatibility_error( + migration, + format!( + "temporary capability '{}' already exists", + definition.temporary_table + ), + )); + } + + execute_step( + connection, + migration, + step_index, + definition.create_temporary_sql, + ) + .await?; + execute_step(connection, migration, step_index, definition.copy_sql).await?; + execute_step(connection, migration, step_index, definition.drop_sql).await?; + execute_step(connection, migration, step_index, definition.rename_sql).await?; + for object in dependent_objects { + execute_step(connection, migration, step_index, &object.sql) + .await + .map_err(|error| { + PersistenceError::Storage(format!( + "{error}; failed while preserving {} capability '{}'", + object.kind, object.name + )) + })?; + } + Ok(()) +} + +async fn table_columns( + connection: &Connection, + migration: &Migration, + table: &str, +) -> Result, PersistenceError> { + let pragma = format!("PRAGMA table_xinfo({})", quote_identifier(table)); + let mut rows = connection + .query(&pragma, ()) + .await + .map_err(|error| inspection_error(migration, "inspect OTS columns", error))?; + let mut columns = BTreeMap::new(); + while let Some(row) = rows + .next() + .await + .map_err(|error| inspection_error(migration, "read OTS column", error))? + { + let name = row + .get::(1) + .map_err(|error| inspection_error(migration, "decode OTS column name", error))?; + let declared_type = row + .get::(2) + .map_err(|error| inspection_error(migration, "decode OTS column type", error))?; + let default = row + .get::>(4) + .map_err(|error| inspection_error(migration, "decode OTS column default", error))?; + columns.insert( + name, + ObservedColumn { + affinity: type_affinity(&declared_type).to_string(), + not_null: row.get::(3).map_err(|error| { + inspection_error(migration, "decode OTS not-null flag", error) + })? != 0, + default: default.map(|value| normalize_default(&value)), + primary_key_position: row.get::(5).map_err(|error| { + inspection_error(migration, "decode OTS primary key", error) + })?, + hidden: row.get::(6).map_err(|error| { + inspection_error(migration, "decode OTS hidden-column flag", error) + })?, + }, + ); + } + Ok(columns) +} + +fn validate_columns( + migration: &Migration, + definition: &OtsRebuildDefinition, + actual: &BTreeMap, + already_updated: bool, +) -> Result<(), PersistenceError> { + let mut expected = definition.required_columns.to_vec(); + let shape = if already_updated { + expected.push(definition.updated_at_column); + "current" + } else { + "pre-upgrade" + }; + if actual.len() != expected.len() { + let expected_names = expected + .iter() + .map(|column| column.name) + .collect::>(); + return Err(compatibility_error( + migration, + format!( + "table '{}' must contain exactly the {shape} columns {expected_names:?}; found {:?}", + definition.table, + actual.keys().collect::>() + ), + )); + } + for expected in expected { + let Some(observed) = actual.get(expected.name) else { + return Err(compatibility_error( + migration, + format!( + "table '{}' is missing required {shape} column '{}'", + definition.table, expected.name + ), + )); + }; + let expected_observed = ObservedColumn { + affinity: expected.affinity.to_string(), + not_null: expected.not_null, + default: expected.default.map(str::to_string), + primary_key_position: expected.primary_key_position, + hidden: 0, + }; + if observed != &expected_observed { + return Err(compatibility_error( + migration, + format!( + "table '{}' column '{}' has incompatible {shape} semantics: expected {expected_observed:?}, found {observed:?}", + definition.table, expected.name + ), + )); + } + } + Ok(()) +} + +async fn validate_no_table_constraints( + connection: &Connection, + migration: &Migration, + definition: &OtsRebuildDefinition, + reject_inbound_references: bool, +) -> Result<(), PersistenceError> { + let table = definition.table; + let mut table_rows = connection + .query( + "SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = ?1", + [table], + ) + .await + .map_err(|error| inspection_error(migration, "inspect OTS table definition", error))?; + let table_sql = table_rows + .next() + .await + .map_err(|error| inspection_error(migration, "read OTS table definition", error))? + .ok_or_else(|| compatibility_error(migration, format!("table '{table}' is missing")))? + .get::(0) + .map_err(|error| inspection_error(migration, "decode OTS table definition", error))?; + if let Some(sequence_index) = + contains_sequence(&table_sql, definition.forbidden_table_sql_sequences) + { + let sequence = definition.forbidden_table_sql_sequences[sequence_index].join(" "); + return Err(compatibility_error( + migration, + format!( + "table '{table}' contains unsupported legacy table semantics matching '{sequence}'" + ), + )); + } + drop(table_rows); + + let index_pragma = format!("PRAGMA index_list({})", quote_identifier(table)); + let mut indexes = connection + .query(&index_pragma, ()) + .await + .map_err(|error| inspection_error(migration, "inspect OTS indexes", error))?; + while let Some(row) = indexes + .next() + .await + .map_err(|error| inspection_error(migration, "read OTS index", error))? + { + let name = row + .get::(1) + .map_err(|error| inspection_error(migration, "decode OTS index name", error))?; + let unique = row + .get::(2) + .map_err(|error| inspection_error(migration, "decode OTS unique flag", error))? + != 0; + let origin = row + .get::(3) + .map_err(|error| inspection_error(migration, "decode OTS index origin", error))?; + if unique && origin == "pk" { + let actual = index_columns(connection, &name).await?; + let expected = expected_primary_key(definition); + if actual != expected { + return Err(compatibility_error( + migration, + format!( + "table '{table}' has incompatible primary key semantics: expected {expected:?}, found {actual:?}" + ), + )); + } + } else if unique { + return Err(compatibility_error( + migration, + format!("table '{table}' has an unsupported legacy unique restriction"), + )); + } + } + drop(indexes); + + let foreign_key_pragma = format!("PRAGMA foreign_key_list({})", quote_identifier(table)); + let mut foreign_keys = connection + .query(&foreign_key_pragma, ()) + .await + .map_err(|error| inspection_error(migration, "inspect OTS foreign keys", error))?; + if foreign_keys + .next() + .await + .map_err(|error| inspection_error(migration, "read OTS foreign key", error))? + .is_some() + { + return Err(compatibility_error( + migration, + format!("table '{table}' has unsupported legacy foreign keys"), + )); + } + drop(foreign_keys); + + if !reject_inbound_references { + return Ok(()); + } + + let mut tables = connection + .query(definition.schema_tables_query, ()) + .await + .map_err(|error| inspection_error(migration, "list tables for OTS references", error))?; + let mut table_names = Vec::new(); + while let Some(row) = tables + .next() + .await + .map_err(|error| inspection_error(migration, "read table for OTS references", error))? + { + table_names.push(row.get::(0).map_err(|error| { + inspection_error(migration, "decode table for OTS references", error) + })?); + } + drop(tables); + for source_table in table_names { + if source_table == table { + continue; + } + let pragma = format!( + "PRAGMA foreign_key_list({})", + quote_identifier(&source_table) + ); + let mut references = connection.query(&pragma, ()).await.map_err(|error| { + inspection_error(migration, "inspect inbound OTS foreign keys", error) + })?; + while let Some(row) = references + .next() + .await + .map_err(|error| inspection_error(migration, "read inbound OTS foreign key", error))? + { + let target = row.get::(2).map_err(|error| { + inspection_error(migration, "decode inbound OTS foreign key", error) + })?; + if target.eq_ignore_ascii_case(table) { + return Err(compatibility_error( + migration, + format!( + "table '{source_table}' has an inbound foreign key to legacy table '{table}'" + ), + )); + } + } + } + Ok(()) +} + +fn expected_primary_key(definition: &OtsRebuildDefinition) -> Vec { + let mut columns = definition + .required_columns + .iter() + .filter(|column| column.primary_key_position > 0) + .collect::>(); + columns.sort_by_key(|column| column.primary_key_position); + columns + .into_iter() + .map(|column| IndexColumn { + name: Some(column.name.to_string()), + descending: false, + collation: Some("binary".to_string()), + }) + .collect() +} + +async fn dependent_objects( + connection: &Connection, + migration: &Migration, + definition: &OtsRebuildDefinition, +) -> Result, PersistenceError> { + let mut rows = connection + .query(definition.dependent_objects_query, [definition.table]) + .await + .map_err(|error| inspection_error(migration, "inspect dependent OTS schema", error))?; + let mut objects = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|error| inspection_error(migration, "read dependent OTS schema", error))? + { + objects.push(DependentObject { + kind: row.get::(0).map_err(|error| { + inspection_error(migration, "decode dependent OTS object kind", error) + })?, + name: row.get::(1).map_err(|error| { + inspection_error(migration, "decode dependent OTS object name", error) + })?, + sql: row.get::(2).map_err(|error| { + inspection_error(migration, "decode dependent OTS object SQL", error) + })?, + }); + } + Ok(objects) +} + +fn quote_identifier(identifier: &str) -> String { + format!("\"{}\"", identifier.replace('"', "\"\"")) +} + +fn inspection_error( + migration: &Migration, + context: &str, + error: libsql::Error, +) -> PersistenceError { + PersistenceError::Storage(format!( + "Turso migration {} ({}) failed while attempting to {context}: {error} ({error:?})", + migration.version, migration.name + )) +} + +fn compatibility_error(migration: &Migration, message: String) -> PersistenceError { + PersistenceError::Storage(format!( + "Turso migration {} ({}) schema compatibility check failed: {message}", + migration.version, migration.name + )) +} diff --git a/crates/temper-store-turso/src/migrations/ots_rebuild_tests.rs b/crates/temper-store-turso/src/migrations/ots_rebuild_tests.rs new file mode 100644 index 000000000..86daa6e59 --- /dev/null +++ b/crates/temper-store-turso/src/migrations/ots_rebuild_tests.rs @@ -0,0 +1,469 @@ +use libsql::Builder; + +use super::runner::migrate; + +#[tokio::test] +async fn compact_legacy_check_constraint_fails_without_mutation() { + let directory = tempfile::tempdir().expect("temporary database directory"); + let database = Builder::new_local(directory.path().join("compact-check.db")) + .build() + .await + .expect("build compact-check database"); + let connection = database.connect().expect("connect compact-check database"); + connection + .execute( + "CREATE TABLE ots_trajectories ( + trajectory_id TEXT PRIMARY KEY, + tenant TEXT NOT NULL, + agent_id TEXT NOT NULL, + session_id TEXT, + outcome TEXT NOT NULL DEFAULT 'unknown', + entity_type TEXT, + turn_count INTEGER NOT NULL DEFAULT 0, + data TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')),CHECK(length(data)>0))", + (), + ) + .await + .expect("legacy OTS table with compact check"); + connection + .execute( + "INSERT INTO ots_trajectories (trajectory_id, tenant, agent_id, data) + VALUES ('trajectory-check', 'tenant-a', 'agent-a', '{}')", + (), + ) + .await + .expect("legacy OTS row"); + + let error = migrate(&connection) + .await + .expect_err("compact CHECK syntax must prevent a destructive rebuild"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains("migration 5"), "{diagnostic}"); + assert!(diagnostic.contains("matching 'CHECK'"), "{diagnostic}"); + + let table_sql = scalar_string( + &connection, + "SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = 'ots_trajectories'", + ) + .await; + assert!(table_sql.contains(",CHECK(length(data)>0)"), "{table_sql}"); + assert_eq!( + scalar_i64( + &connection, + "SELECT COUNT(*) FROM ots_trajectories WHERE trajectory_id = 'trajectory-check'" + ) + .await, + 1 + ); + assert_eq!( + scalar_i64( + &connection, + "SELECT COUNT(*) FROM pragma_table_info('ots_trajectories') + WHERE name IN ('persistence_status', 'persist_attempts', 'last_error', 'updated_at')" + ) + .await, + 0 + ); + assert_eq!( + scalar_i64(&connection, "SELECT COUNT(*) FROM temper_schema_migrations").await, + 4 + ); +} + +#[tokio::test] +async fn current_shape_check_constraint_fails_without_ledgering_migration_five() { + let directory = tempfile::tempdir().expect("temporary database directory"); + let database = Builder::new_local(directory.path().join("current-check.db")) + .build() + .await + .expect("build current-check database"); + let connection = database.connect().expect("connect current-check database"); + connection + .execute( + "CREATE TABLE ots_trajectories ( + trajectory_id TEXT PRIMARY KEY, + tenant TEXT NOT NULL, + agent_id TEXT NOT NULL, + session_id TEXT, + outcome TEXT NOT NULL DEFAULT 'unknown', + entity_type TEXT, + turn_count INTEGER NOT NULL DEFAULT 0, + data TEXT NOT NULL, + persistence_status TEXT NOT NULL DEFAULT 'persisted', + persist_attempts INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + CHECK (data <> '{}') + )", + (), + ) + .await + .expect("current OTS table with restrictive check"); + connection + .execute( + "INSERT INTO ots_trajectories (trajectory_id, tenant, agent_id, data) + VALUES ('trajectory-current-check', 'tenant-a', 'agent-a', '{\"turns\":1}')", + (), + ) + .await + .expect("current OTS row"); + let before = scalar_string( + &connection, + "SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = 'ots_trajectories'", + ) + .await; + + let error = migrate(&connection) + .await + .expect_err("a current-shape CHECK must prevent migration readiness"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains("migration 5"), "{diagnostic}"); + assert!(diagnostic.contains("matching 'CHECK'"), "{diagnostic}"); + assert_eq!( + scalar_string( + &connection, + "SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = 'ots_trajectories'", + ) + .await, + before + ); + assert_eq!( + scalar_i64( + &connection, + "SELECT COUNT(*) FROM ots_trajectories + WHERE trajectory_id = 'trajectory-current-check'" + ) + .await, + 1 + ); + assert_eq!( + scalar_i64(&connection, "SELECT COUNT(*) FROM temper_schema_migrations").await, + 4 + ); +} + +#[tokio::test] +async fn current_shape_short_generated_column_fails_without_mutation() { + let directory = tempfile::tempdir().expect("temporary database directory"); + let database = Builder::new_local(directory.path().join("current-generated.db")) + .build() + .await + .expect("build current-generated database"); + let connection = database + .connect() + .expect("connect current-generated database"); + connection + .execute( + "CREATE TABLE ots_trajectories ( + trajectory_id TEXT PRIMARY KEY, + tenant TEXT NOT NULL, + agent_id TEXT NOT NULL, + session_id TEXT, + outcome TEXT NOT NULL DEFAULT 'unknown', + entity_type TEXT, + turn_count INTEGER NOT NULL DEFAULT 0, + data TEXT NOT NULL, + persistence_status TEXT NOT NULL DEFAULT 'persisted', + persist_attempts INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')), + risky TEXT AS (json_extract(data, '$.required')) NOT NULL + )", + (), + ) + .await + .expect("current OTS table with short-form generated column"); + connection + .execute( + "INSERT INTO ots_trajectories ( + trajectory_id, tenant, agent_id, data + ) VALUES ( + 'trajectory-generated', 'tenant-a', 'agent-a', '{\"required\":\"present\"}' + )", + (), + ) + .await + .expect("current OTS row satisfying generated restriction"); + let before = scalar_string( + &connection, + "SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = 'ots_trajectories'", + ) + .await; + + let error = migrate(&connection) + .await + .expect_err("a current-shape short-form generated column must prevent readiness"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains("migration 5"), "{diagnostic}"); + assert!(diagnostic.contains("risky"), "{diagnostic}"); + assert_eq!( + scalar_string( + &connection, + "SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = 'ots_trajectories'", + ) + .await, + before + ); + assert_eq!( + scalar_i64( + &connection, + "SELECT COUNT(*) FROM ots_trajectories + WHERE trajectory_id = 'trajectory-generated'" + ) + .await, + 1 + ); + assert_eq!( + scalar_i64(&connection, "SELECT COUNT(*) FROM temper_schema_migrations").await, + 4 + ); +} + +#[tokio::test] +async fn descending_primary_key_fails_before_legacy_ots_rebuild() { + let directory = tempfile::tempdir().expect("temporary database directory"); + let database = Builder::new_local(directory.path().join("descending-primary-key.db")) + .build() + .await + .expect("build descending-primary-key database"); + let connection = database + .connect() + .expect("connect descending-primary-key database"); + create_legacy_ots(&connection, "PRIMARY KEY DESC").await; + connection + .execute( + "INSERT INTO ots_trajectories (trajectory_id, tenant, agent_id, data) + VALUES ('trajectory-desc', 'tenant-a', 'agent-a', '{}')", + (), + ) + .await + .expect("legacy OTS row with descending primary key"); + let before = scalar_string( + &connection, + "SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = 'ots_trajectories'", + ) + .await; + + let error = migrate(&connection) + .await + .expect_err("descending primary-key semantics must prevent a rebuild"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains("migration 5"), "{diagnostic}"); + assert!(diagnostic.contains("primary key semantics"), "{diagnostic}"); + assert!(diagnostic.contains("descending: true"), "{diagnostic}"); + assert_eq!( + scalar_string( + &connection, + "SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = 'ots_trajectories'", + ) + .await, + before + ); + assert_eq!( + scalar_i64( + &connection, + "SELECT COUNT(*) FROM ots_trajectories WHERE trajectory_id = 'trajectory-desc'" + ) + .await, + 1 + ); + assert_eq!( + scalar_i64(&connection, "SELECT COUNT(*) FROM temper_schema_migrations").await, + 4 + ); +} + +#[tokio::test] +async fn legal_sqlitex_child_foreign_key_fails_before_cascade_data_loss() { + let directory = tempfile::tempdir().expect("temporary database directory"); + let database = Builder::new_local(directory.path().join("sqlitex-child.db")) + .build() + .await + .expect("build sqliteX-child database"); + let connection = database.connect().expect("connect sqliteX-child database"); + connection + .execute("PRAGMA foreign_keys = ON", ()) + .await + .expect("enable foreign keys"); + create_legacy_ots(&connection, "PRIMARY KEY").await; + connection + .execute( + "CREATE TABLE sqliteX_child ( + id TEXT PRIMARY KEY, + trajectory_id TEXT NOT NULL REFERENCES ots_trajectories(trajectory_id) + ON DELETE CASCADE + )", + (), + ) + .await + .expect("legal child table with inbound OTS foreign key"); + connection + .execute( + "INSERT INTO ots_trajectories (trajectory_id, tenant, agent_id, data) + VALUES ('trajectory-parent', 'tenant-a', 'agent-a', '{}')", + (), + ) + .await + .expect("legacy OTS parent row"); + connection + .execute( + "INSERT INTO sqliteX_child (id, trajectory_id) + VALUES ('child-1', 'trajectory-parent')", + (), + ) + .await + .expect("inbound child row"); + let before = scalar_string( + &connection, + "SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = 'ots_trajectories'", + ) + .await; + + let error = migrate(&connection) + .await + .expect_err("an inbound FK from a legal sqliteX name must prevent a rebuild"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains("migration 5"), "{diagnostic}"); + assert!(diagnostic.contains("sqliteX_child"), "{diagnostic}"); + assert!(diagnostic.contains("inbound foreign key"), "{diagnostic}"); + assert_eq!( + scalar_string( + &connection, + "SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = 'ots_trajectories'", + ) + .await, + before + ); + assert_eq!( + scalar_i64(&connection, "SELECT COUNT(*) FROM sqliteX_child").await, + 1 + ); + assert_eq!( + scalar_i64( + &connection, + "SELECT COUNT(*) FROM ots_trajectories WHERE trajectory_id = 'trajectory-parent'" + ) + .await, + 1 + ); + assert_eq!( + scalar_i64(&connection, "SELECT COUNT(*) FROM temper_schema_migrations").await, + 4 + ); +} + +#[tokio::test] +async fn differently_cased_ots_trigger_owner_is_preserved_when_contract_rejects() { + let directory = tempfile::tempdir().expect("temporary database directory"); + let database = Builder::new_local(directory.path().join("case-folded-ots-trigger.db")) + .build() + .await + .expect("build case-folded-OTS-trigger database"); + let connection = database + .connect() + .expect("connect case-folded-OTS-trigger database"); + create_legacy_ots(&connection, "PRIMARY KEY").await; + connection + .execute( + "CREATE TRIGGER reject_case_folded_ots BEFORE INSERT ON OTS_TRAJECTORIES + BEGIN SELECT RAISE(FAIL, 'blocked'); END", + (), + ) + .await + .expect("create blocking trigger with differently cased OTS owner"); + let before = scalar_string( + &connection, + "SELECT sql FROM sqlite_schema + WHERE type = 'trigger' AND name = 'reject_case_folded_ots'", + ) + .await; + + let error = migrate(&connection) + .await + .expect_err("the preserved trigger must fail the supported audit contract"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains("migration 5"), "{diagnostic}"); + assert!( + diagnostic.contains("reject_case_folded_ots"), + "{diagnostic}" + ); + assert!( + diagnostic.contains("unsupported executable trigger extension"), + "{diagnostic}" + ); + assert_eq!( + scalar_string( + &connection, + "SELECT sql FROM sqlite_schema + WHERE type = 'trigger' AND name = 'reject_case_folded_ots'", + ) + .await, + before + ); + assert_eq!( + scalar_i64( + &connection, + "SELECT COUNT(*) FROM pragma_table_info('ots_trajectories') + WHERE name = 'updated_at'" + ) + .await, + 0, + "the rejected contract must roll back the destructive rebuild" + ); + assert_eq!( + scalar_i64(&connection, "SELECT COUNT(*) FROM temper_schema_migrations").await, + 4 + ); +} + +async fn create_legacy_ots(connection: &libsql::Connection, primary_key: &str) { + let sql = format!( + "CREATE TABLE ots_trajectories ( + trajectory_id TEXT {primary_key}, + tenant TEXT NOT NULL, + agent_id TEXT NOT NULL, + session_id TEXT, + outcome TEXT NOT NULL DEFAULT 'unknown', + entity_type TEXT, + turn_count INTEGER NOT NULL DEFAULT 0, + data TEXT NOT NULL, + persistence_status TEXT NOT NULL DEFAULT 'persisted', + persist_attempts INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')) + )" + ); + connection + .execute(&sql, ()) + .await + .expect("create legacy OTS table"); +} + +async fn scalar_string(connection: &libsql::Connection, sql: &str) -> String { + let mut rows = connection + .query(sql, ()) + .await + .expect("query string scalar"); + rows.next() + .await + .expect("read string scalar") + .expect("string scalar row") + .get::(0) + .expect("string scalar value") +} + +async fn scalar_i64(connection: &libsql::Connection, sql: &str) -> i64 { + let mut rows = connection + .query(sql, ()) + .await + .expect("query integer scalar"); + rows.next() + .await + .expect("read integer scalar") + .expect("integer scalar row") + .get::(0) + .expect("integer scalar value") +} diff --git a/crates/temper-store-turso/src/migrations/remote_probe_tests.rs b/crates/temper-store-turso/src/migrations/remote_probe_tests.rs new file mode 100644 index 000000000..6ce3aa479 --- /dev/null +++ b/crates/temper-store-turso/src/migrations/remote_probe_tests.rs @@ -0,0 +1,91 @@ +use libsql::Builder; + +use super::runner::migrate; + +const REMOTE_URL_ENV: &str = "TEMPER_HRANA_MIGRATION_TEST_URL"; +const REMOTE_TOKEN_ENV: &str = "TEMPER_HRANA_MIGRATION_TEST_TOKEN"; + +#[tokio::test] +#[ignore = "requires an isolated Hrana endpoint via TEMPER_HRANA_MIGRATION_TEST_URL"] +async fn remote_final_ots_probe_is_atomic_and_side_effect_free() { + let url = std::env::var(REMOTE_URL_ENV).expect("isolated Hrana test URL"); + let token = std::env::var(REMOTE_TOKEN_ENV).expect("isolated Hrana test token"); + assert!(url.starts_with("http"), "isolated Hrana endpoint URL"); + + let initial_database = Builder::new_remote(url.clone(), token.clone()) + .build() + .await + .expect("build isolated remote database client"); + let setup = initial_database + .connect() + .expect("connect isolated remote database"); + migrate(&setup) + .await + .expect("migrate isolated remote database"); + drop(setup); + let setup = initial_database + .connect() + .expect("reconnect remote setup after migration"); + let ots_rows_before = scalar_i64(&setup, "SELECT COUNT(*) FROM ots_trajectories").await; + setup + .execute( + "CREATE TABLE arn242_remote_probe_audit ( + trajectory_id TEXT NOT NULL + )", + (), + ) + .await + .expect("create remote probe audit table"); + setup + .execute( + "CREATE TRIGGER arn242_remote_probe_audit_trigger + AFTER INSERT ON ots_trajectories + BEGIN + INSERT INTO arn242_remote_probe_audit (trajectory_id) + VALUES (NEW.trajectory_id); + END", + (), + ) + .await + .expect("create benign remote OTS audit trigger"); + + let replay_database = Builder::new_remote(url, token) + .build() + .await + .expect("build remote replay client"); + let replay = replay_database + .connect() + .expect("connect remote replay client"); + let reopened = migrate(&replay).await; + drop(replay); + let inspection = initial_database + .connect() + .expect("reconnect remote inspection after replay"); + let ots_rows_after = scalar_i64(&inspection, "SELECT COUNT(*) FROM ots_trajectories").await; + let audit_rows = scalar_i64( + &inspection, + "SELECT COUNT(*) FROM arn242_remote_probe_audit", + ) + .await; + let reopen_error = reopened.as_ref().err().map(ToString::to_string); + + assert!( + reopened.is_ok() && ots_rows_after == ots_rows_before && audit_rows == 0, + "remote head verification must succeed without durable probe effects: \ + reopen_error={reopen_error:?}, ots_rows_before={ots_rows_before}, \ + ots_rows_after={ots_rows_after}, audit_rows={audit_rows}" + ); +} + +async fn scalar_i64(connection: &libsql::Connection, sql: &str) -> i64 { + let mut rows = connection + .query(sql, ()) + .await + .expect("query remote scalar"); + rows.next() + .await + .expect("read remote scalar") + .expect("remote scalar row") + .get::(0) + .expect("decode remote scalar") +} diff --git a/crates/temper-store-turso/src/migrations/runner.rs b/crates/temper-store-turso/src/migrations/runner.rs new file mode 100644 index 000000000..368351988 --- /dev/null +++ b/crates/temper-store-turso/src/migrations/runner.rs @@ -0,0 +1,494 @@ +use std::collections::BTreeSet; + +use libsql::{Builder, Connection, TransactionBehavior, params}; +use temper_runtime::persistence::PersistenceError; + +use super::catalog::{MIGRATIONS, Migration, MigrationStep}; +use super::ledger::{CREATE_MIGRATION_LEDGER, validate_ledger_schema}; +use super::ots_rebuild::rebuild_ots_trajectories; +use super::schema_snapshot::{SchemaSnapshot, capture_schema, verify_schema}; + +#[derive(Clone, Copy, Debug, Default)] +pub(super) struct FaultInjection { + pub after_step: Option<(u32, usize)>, + pub ddl_error_at: Option<(u32, usize)>, +} + +#[derive(Debug)] +struct LedgerRow { + version: u32, + name: String, + checksum: String, +} + +#[derive(Debug)] +struct ExpectedMigration { + snapshot: SchemaSnapshot, + checksum: String, +} + +pub(crate) async fn migrate(connection: &Connection) -> Result<(), PersistenceError> { + run_migrations(connection, MIGRATIONS, FaultInjection::default()).await +} + +#[cfg(test)] +pub(super) async fn migrate_prefix( + connection: &Connection, + migration_count: usize, + fault: FaultInjection, +) -> Result<(), PersistenceError> { + assert!(migration_count <= MIGRATIONS.len()); + run_migrations(connection, &MIGRATIONS[..migration_count], fault).await +} + +#[cfg(test)] +pub(super) async fn migrate_catalog( + connection: &Connection, + catalog: &[Migration], +) -> Result<(), PersistenceError> { + run_migrations(connection, catalog, FaultInjection::default()).await +} + +async fn run_migrations( + connection: &Connection, + catalog: &[Migration], + fault: FaultInjection, +) -> Result<(), PersistenceError> { + validate_catalog(catalog)?; + let expected_migrations = build_expected_migrations(catalog).await?; + ensure_ledger(connection).await?; + validate_ledger_rows( + &load_ledger(connection).await?, + catalog, + &expected_migrations, + )?; + for (index, migration) in catalog.iter().enumerate() { + apply_migration( + connection, + catalog, + migration, + &expected_migrations, + index, + fault, + ) + .await?; + } + + let transaction = connection + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .map_err(|error| { + migration_sql_error("begin catalog-head verification transaction", error) + })?; + let outcome = async { + let final_ledger = load_ledger(&transaction).await?; + validate_ledger_rows(&final_ledger, catalog, &expected_migrations)?; + require_ledger_length(&final_ledger, catalog.len(), "after migration run")?; + if let Some((migration, expected)) = catalog.last().zip(expected_migrations.last()) { + verify_schema(&transaction, &expected.snapshot) + .await + .map_err(|error| { + migration_context(migration, "verify catalog head schema", error) + })?; + } + Ok(()) + } + .await; + finish_transaction(transaction, outcome, "verify catalog head schema").await +} + +async fn ensure_ledger(connection: &Connection) -> Result<(), PersistenceError> { + let transaction = connection + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .map_err(|error| migration_sql_error("begin migration-ledger transaction", error))?; + let outcome = async { + transaction + .execute(CREATE_MIGRATION_LEDGER, ()) + .await + .map_err(|error| migration_sql_error("create migration ledger", error))?; + validate_ledger_schema(&transaction).await + } + .await; + finish_transaction(transaction, outcome, "create migration ledger").await +} + +async fn apply_migration( + connection: &Connection, + catalog: &[Migration], + migration: &Migration, + expected_migrations: &[ExpectedMigration], + migration_index: usize, + fault: FaultInjection, +) -> Result<(), PersistenceError> { + let transaction = connection + .transaction_with_behavior(TransactionBehavior::Immediate) + .await + .map_err(|error| { + migration_sql_error( + &format!("begin migration {} ({})", migration.version, migration.name), + error, + ) + })?; + + let outcome = async { + let ledger = load_ledger(&transaction).await?; + validate_ledger_rows(&ledger, catalog, expected_migrations)?; + if ledger + .last() + .is_some_and(|row| row.version >= migration.version) + { + return Ok(()); + } + + for (step_index, step) in migration.steps.iter().enumerate() { + if fault.ddl_error_at == Some((migration.version, step_index)) { + execute_step( + &transaction, + migration, + step_index, + "ALTER TABLE __temper_injected_missing_table ADD COLUMN value TEXT", + ) + .await?; + } + apply_step(&transaction, migration, step_index, step).await?; + if fault.after_step == Some((migration.version, step_index)) { + return Err(PersistenceError::Storage(format!( + "injected migration interruption after version {} step {step_index}", + migration.version + ))); + } + } + + let expected = &expected_migrations[migration_index]; + verify_schema(&transaction, &expected.snapshot) + .await + .map_err(|error| migration_context(migration, "verify schema", error))?; + let inserted = transaction + .execute( + "INSERT INTO temper_schema_migrations (version, name, checksum) + VALUES (?1, ?2, ?3)", + params![ + migration.version as i64, + migration.name, + expected.checksum.as_str() + ], + ) + .await + .map_err(|error| { + migration_sql_error( + &format!( + "record migration {} ({})", + migration.version, migration.name + ), + error, + ) + })?; + if inserted != 1 { + return Err(PersistenceError::Storage(format!( + "Turso migration ledger insert for version {} ({}) affected {inserted} rows; expected 1", + migration.version, migration.name + ))); + } + let retained_ledger = load_ledger(&transaction).await?; + validate_ledger_rows(&retained_ledger, catalog, expected_migrations)?; + require_ledger_length( + &retained_ledger, + migration_index + 1, + &format!("after recording migration {}", migration.version), + )?; + Ok(()) + } + .await; + + finish_transaction( + transaction, + outcome, + &format!("apply migration {} ({})", migration.version, migration.name), + ) + .await +} + +async fn finish_transaction( + transaction: libsql::Transaction, + outcome: Result<(), PersistenceError>, + context: &str, +) -> Result<(), PersistenceError> { + match outcome { + Ok(()) => transaction + .commit() + .await + .map_err(|error| migration_sql_error(&format!("commit {context}"), error)), + Err(error) => match transaction.rollback().await { + Ok(()) => Err(error), + Err(rollback_error) => Err(PersistenceError::Storage(format!( + "{error}; rollback also failed while attempting to {context}: {rollback_error} ({rollback_error:?})" + ))), + }, + } +} + +async fn apply_step( + connection: &Connection, + migration: &Migration, + step_index: usize, + step: &MigrationStep, +) -> Result<(), PersistenceError> { + match step { + MigrationStep::Sql(sql) => execute_step(connection, migration, step_index, sql).await, + MigrationStep::AddColumn { table, column, sql } => { + require_table_kind(connection, migration, step_index, table).await?; + if column_exists(connection, table, column).await? { + Ok(()) + } else { + execute_step(connection, migration, step_index, sql).await + } + } + MigrationStep::RebuildOtsTrajectories => { + rebuild_ots_trajectories(connection, migration, step_index).await + } + } +} + +pub(super) async fn execute_step( + connection: &Connection, + migration: &Migration, + step_index: usize, + sql: &str, +) -> Result<(), PersistenceError> { + connection + .execute(sql, ()) + .await + .map(|_| ()) + .map_err(|error| { + migration_sql_error( + &format!( + "apply migration {} ({}) step {step_index}: {}", + migration.version, + migration.name, + sql.split_whitespace().take(8).collect::>().join(" ") + ), + error, + ) + }) +} + +async fn column_exists( + connection: &Connection, + table: &str, + column: &str, +) -> Result { + Ok(table_columns(connection, table).await?.contains(column)) +} + +async fn require_table_kind( + connection: &Connection, + migration: &Migration, + step_index: usize, + table: &str, +) -> Result<(), PersistenceError> { + let kind = schema_object_kind(connection, table).await?; + if kind.as_deref() == Some("table") { + return Ok(()); + } + Err(PersistenceError::Storage(format!( + "Turso migration {} ({}) step {step_index} schema compatibility check failed: capability '{table}' must be a table, found {}", + migration.version, + migration.name, + kind.as_deref().unwrap_or("no schema object") + ))) +} + +pub(super) async fn table_columns( + connection: &Connection, + table: &str, +) -> Result, PersistenceError> { + let pragma = format!("PRAGMA table_xinfo({})", quote_identifier(table)); + let mut rows = connection + .query(&pragma, ()) + .await + .map_err(|error| migration_sql_error(&format!("inspect table '{table}'"), error))?; + let mut columns = BTreeSet::new(); + while let Some(row) = rows + .next() + .await + .map_err(|error| migration_sql_error(&format!("read table '{table}'"), error))? + { + columns.insert(row.get::(1).map_err(|error| { + migration_sql_error(&format!("decode column for table '{table}'"), error) + })?); + } + Ok(columns) +} + +pub(super) async fn schema_object_kind( + connection: &Connection, + name: &str, +) -> Result, PersistenceError> { + let mut rows = connection + .query( + "SELECT type FROM sqlite_schema WHERE name = ?1 ORDER BY type LIMIT 1", + [name], + ) + .await + .map_err(|error| migration_sql_error("inspect schema object", error))?; + rows.next() + .await + .map_err(|error| migration_sql_error("read schema object", error))? + .map(|row| { + row.get::(0) + .map_err(|error| migration_sql_error("decode schema object", error)) + }) + .transpose() +} + +async fn build_expected_migrations( + catalog: &[Migration], +) -> Result, PersistenceError> { + let database = Builder::new_local(":memory:") + .build() + .await + .map_err(|error| migration_sql_error("build reference schema database", error))?; + let connection = database + .connect() + .map_err(|error| migration_sql_error("connect to reference schema database", error))?; + let mut expected_migrations = Vec::with_capacity(catalog.len()); + for migration in catalog { + for (step_index, step) in migration.steps.iter().enumerate() { + apply_step(&connection, migration, step_index, step).await?; + } + let snapshot = capture_schema(&connection).await?; + let checksum = migration.checksum(&snapshot.manifest()); + expected_migrations.push(ExpectedMigration { snapshot, checksum }); + } + Ok(expected_migrations) +} + +#[cfg(test)] +pub(super) async fn expected_checksums() -> Result, PersistenceError> { + Ok(build_expected_migrations(MIGRATIONS) + .await? + .into_iter() + .map(|migration| migration.checksum) + .collect()) +} + +async fn load_ledger(connection: &Connection) -> Result, PersistenceError> { + let mut rows = connection + .query( + "SELECT version, name, checksum FROM temper_schema_migrations ORDER BY version", + (), + ) + .await + .map_err(|error| migration_sql_error("load migration ledger", error))?; + let mut ledger = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|error| migration_sql_error("read migration-ledger row", error))? + { + let version = row + .get::(0) + .map_err(|error| migration_sql_error("decode migration version", error))?; + let version = u32::try_from(version).map_err(|_| { + PersistenceError::Storage(format!( + "Turso migration ledger contains invalid version {version}" + )) + })?; + ledger.push(LedgerRow { + version, + name: row + .get::(1) + .map_err(|error| migration_sql_error("decode migration name", error))?, + checksum: row + .get::(2) + .map_err(|error| migration_sql_error("decode migration checksum", error))?, + }); + } + Ok(ledger) +} + +fn validate_catalog(catalog: &[Migration]) -> Result<(), PersistenceError> { + for (index, migration) in catalog.iter().enumerate() { + let expected = (index + 1) as u32; + if migration.version != expected { + return Err(PersistenceError::Storage(format!( + "Turso migration catalog is not contiguous: expected version {expected}, found {}", + migration.version + ))); + } + } + Ok(()) +} + +fn validate_ledger_rows( + ledger: &[LedgerRow], + catalog: &[Migration], + expected_migrations: &[ExpectedMigration], +) -> Result<(), PersistenceError> { + for (index, row) in ledger.iter().enumerate() { + let expected_version = (index + 1) as u32; + if row.version != expected_version { + return Err(PersistenceError::Storage(format!( + "Turso migration ledger has a version gap: expected {expected_version}, found {}", + row.version + ))); + } + let Some(migration) = catalog.get(index) else { + return Err(PersistenceError::Storage(format!( + "Turso database schema version {} is newer than this binary supports (latest supported version {})", + row.version, + catalog.last().map_or(0, |migration| migration.version) + ))); + }; + if row.name != migration.name { + return Err(PersistenceError::Storage(format!( + "Turso migration ledger name mismatch at version {}: expected '{}', found '{}'", + row.version, migration.name, row.name + ))); + } + let expected_checksum = &expected_migrations[index].checksum; + if &row.checksum != expected_checksum { + return Err(PersistenceError::Storage(format!( + "Turso migration ledger checksum mismatch at version {} ({}): expected {}, found {}", + row.version, row.name, expected_checksum, row.checksum + ))); + } + } + Ok(()) +} + +fn require_ledger_length( + ledger: &[LedgerRow], + expected: usize, + context: &str, +) -> Result<(), PersistenceError> { + if ledger.len() == expected { + return Ok(()); + } + Err(PersistenceError::Storage(format!( + "Turso migration ledger is incomplete {context}: expected {expected} rows, found {}", + ledger.len() + ))) +} + +fn quote_identifier(identifier: &str) -> String { + format!("\"{}\"", identifier.replace('"', "\"\"")) +} + +fn migration_sql_error(context: &str, error: libsql::Error) -> PersistenceError { + PersistenceError::Storage(format!( + "Turso schema migration failed while attempting to {context}: {error} ({error:?})" + )) +} + +fn migration_context( + migration: &Migration, + context: &str, + error: PersistenceError, +) -> PersistenceError { + PersistenceError::Storage(format!( + "Turso migration {} ({}) failed while attempting to {context}: {error}", + migration.version, migration.name + )) +} diff --git a/crates/temper-store-turso/src/migrations/schema_manifest.rs b/crates/temper-store-turso/src/migrations/schema_manifest.rs new file mode 100644 index 000000000..4333565f3 --- /dev/null +++ b/crates/temper-store-turso/src/migrations/schema_manifest.rs @@ -0,0 +1,111 @@ +use super::schema_snapshot::{IndexColumn, SchemaSnapshot}; +use super::schema_sql::RESTRICTED_TABLE_SEQUENCES; +use super::schema_verify::{EXTRA_COLUMN_POLICY, EXTRA_INDEX_POLICY, TRIGGER_POLICY}; + +pub(super) fn canonical_manifest(snapshot: &SchemaSnapshot) -> String { + let mut manifest = String::new(); + part(&mut manifest, "temper-schema-capability-manifest-v1"); + part(&mut manifest, EXTRA_COLUMN_POLICY); + part(&mut manifest, EXTRA_INDEX_POLICY); + part(&mut manifest, TRIGGER_POLICY); + count(&mut manifest, RESTRICTED_TABLE_SEQUENCES.len()); + for (name, sequence) in RESTRICTED_TABLE_SEQUENCES { + part(&mut manifest, name); + count(&mut manifest, sequence.len()); + for token in *sequence { + part(&mut manifest, token); + } + } + count(&mut manifest, snapshot.tables.len()); + for (table_name, table) in &snapshot.tables { + part(&mut manifest, table_name); + count(&mut manifest, table.columns.len()); + for (column_name, column) in &table.columns { + part(&mut manifest, column_name); + part(&mut manifest, &column.affinity); + boolean(&mut manifest, column.not_null); + optional(&mut manifest, column.default.as_deref()); + integer(&mut manifest, column.primary_key_position); + integer(&mut manifest, column.hidden); + } + + count(&mut manifest, table.unique_keys.len()); + for key in &table.unique_keys { + boolean(&mut manifest, key.partial); + index_columns(&mut manifest, &key.columns); + optional(&mut manifest, key.predicate.as_deref()); + } + + count(&mut manifest, table.foreign_keys.len()); + for foreign_key in &table.foreign_keys { + integer(&mut manifest, foreign_key.id); + integer(&mut manifest, foreign_key.sequence); + part(&mut manifest, &foreign_key.target_table); + part(&mut manifest, &foreign_key.source_column); + optional(&mut manifest, foreign_key.target_column.as_deref()); + part(&mut manifest, &foreign_key.on_update); + part(&mut manifest, &foreign_key.on_delete); + part(&mut manifest, &foreign_key.match_kind); + } + + count(&mut manifest, table.restricted_semantics.len()); + for semantic in &table.restricted_semantics { + part(&mut manifest, semantic); + } + } + + count(&mut manifest, snapshot.indexes.len()); + for (index_name, index) in &snapshot.indexes { + part(&mut manifest, index_name); + part(&mut manifest, &index.table); + boolean(&mut manifest, index.unique); + boolean(&mut manifest, index.partial); + index_columns(&mut manifest, &index.columns); + optional(&mut manifest, index.predicate.as_deref()); + } + + count(&mut manifest, snapshot.triggers.len()); + for (trigger_name, trigger) in &snapshot.triggers { + part(&mut manifest, trigger_name); + part(&mut manifest, &trigger.table); + part(&mut manifest, &trigger.definition); + } + manifest +} + +fn index_columns(manifest: &mut String, columns: &[IndexColumn]) { + count(manifest, columns.len()); + for column in columns { + optional(manifest, column.name.as_deref()); + boolean(manifest, column.descending); + optional(manifest, column.collation.as_deref()); + } +} + +fn optional(manifest: &mut String, value: Option<&str>) { + match value { + Some(value) => { + part(manifest, "some"); + part(manifest, value); + } + None => part(manifest, "none"), + } +} + +fn boolean(manifest: &mut String, value: bool) { + part(manifest, if value { "true" } else { "false" }); +} + +fn integer(manifest: &mut String, value: i64) { + part(manifest, &value.to_string()); +} + +fn count(manifest: &mut String, value: usize) { + part(manifest, &value.to_string()); +} + +fn part(manifest: &mut String, value: &str) { + manifest.push_str(&value.len().to_string()); + manifest.push(':'); + manifest.push_str(value); +} diff --git a/crates/temper-store-turso/src/migrations/schema_ots_probe.rs b/crates/temper-store-turso/src/migrations/schema_ots_probe.rs new file mode 100644 index 000000000..eb3eebbb8 --- /dev/null +++ b/crates/temper-store-turso/src/migrations/schema_ots_probe.rs @@ -0,0 +1,376 @@ +use libsql::{Connection, params}; +use temper_runtime::persistence::PersistenceError; + +use super::schema_ots_trigger::validate_ots_audit_trigger_contracts; +use super::schema_snapshot::compatibility_error; +use super::schema_trigger::TriggerCapability; +use crate::store::ots::{ + ENQUEUE_OTS_TRAJECTORY_SQL, MARK_OTS_TRAJECTORY_FAILED_SQL, MARK_OTS_TRAJECTORY_PERSISTED_SQL, + PERSIST_OTS_TRAJECTORY_SQL, +}; + +const OTS_PROBE_FAILURE: &str = "trigger probe failure"; + +#[derive(Debug, Eq, PartialEq)] +struct OtsProbeState { + tenant: String, + agent_id: String, + session_id: String, + outcome: String, + turn_count: i64, + data: String, + persistence_status: String, + persist_attempts: i64, + last_error: Option, +} + +struct OtsProbeIdentity { + persisted_id: String, + queued_id: String, + tenant: String, + agent_id: String, +} + +type ExpectedOtsProbeState<'a> = ( + &'a str, + &'a str, + i64, + &'a str, + &'a str, + i64, + Option<&'a str>, +); + +pub(super) async fn validate_legacy_ots_triggers( + connection: &Connection, + triggers: &[(&str, &TriggerCapability)], +) -> Result<(), PersistenceError> { + validate_ots_audit_trigger_contracts(connection, triggers).await?; + let trigger_names = triggers.iter().map(|(name, _)| *name).collect::>(); + connection + .execute("SAVEPOINT temper_verify_ots_triggers", ()) + .await + .map_err(|error| schema_query_error("start OTS trigger probe", error))?; + let outcome = probe_ots_trigger_writes(connection).await; + let rollback = connection + .execute("ROLLBACK TO SAVEPOINT temper_verify_ots_triggers", ()) + .await; + if let Err(error) = rollback { + return Err(schema_query_error("roll back OTS trigger probe", error)); + } + let release = connection + .execute("RELEASE SAVEPOINT temper_verify_ots_triggers", ()) + .await; + if let Err(error) = release { + return Err(schema_query_error("release OTS trigger probe", error)); + } + outcome.map_err(|error| { + compatibility_error(format!( + "table 'ots_trajectories' has executable trigger extension(s) {trigger_names:?} that reject a rollback-only production persist/enqueue/status-transition probe: {error}" + )) + }) +} + +async fn probe_ots_trigger_writes(connection: &Connection) -> Result<(), PersistenceError> { + let mut identity_rows = connection + .query( + "SELECT lower(hex(randomblob(16))), lower(hex(randomblob(16))), + lower(hex(randomblob(16))), lower(hex(randomblob(16)))", + (), + ) + .await + .map_err(|error| schema_query_error("generate OTS trigger probe ids", error))?; + let row = identity_rows + .next() + .await + .map_err(|error| schema_query_error("read OTS trigger probe id", error))? + .ok_or_else(|| compatibility_error("OTS trigger probe id query returned no row".into()))?; + let identity = OtsProbeIdentity { + persisted_id: row + .get::(0) + .map_err(|error| schema_query_error("decode persisted OTS probe id", error))?, + queued_id: row + .get::(1) + .map_err(|error| schema_query_error("decode queued OTS probe id", error))?, + tenant: row + .get::(2) + .map_err(|error| schema_query_error("decode OTS probe tenant", error))?, + agent_id: row + .get::(3) + .map_err(|error| schema_query_error("decode OTS probe agent", error))?, + }; + drop(identity_rows); + + connection + .execute( + PERSIST_OTS_TRAJECTORY_SQL, + params![ + identity.persisted_id.clone(), + identity.tenant.clone(), + identity.agent_id.clone(), + "persist-session".to_string(), + "persist-outcome".to_string(), + 1_i64, + "{\"stage\":\"persist\"}".to_string(), + ], + ) + .await + .map_err(|error| schema_query_error("probe OTS persisted insert", error))?; + require_ots_probe_state( + connection, + &identity.persisted_id, + expected_ots_probe_state( + &identity, + ( + "persist-session", + "persist-outcome", + 1, + "{\"stage\":\"persist\"}", + "persisted", + 0, + None, + ), + ), + "persist", + ) + .await?; + + connection + .execute( + PERSIST_OTS_TRAJECTORY_SQL, + params![ + identity.persisted_id.clone(), + identity.tenant.clone(), + identity.agent_id.clone(), + "persist-replacement-session".to_string(), + "persist-replacement-outcome".to_string(), + 2_i64, + "{\"stage\":\"persist-replacement\"}".to_string(), + ], + ) + .await + .map_err(|error| schema_query_error("probe OTS persisted replacement", error))?; + require_ots_probe_state( + connection, + &identity.persisted_id, + expected_ots_probe_state( + &identity, + ( + "persist-replacement-session", + "persist-replacement-outcome", + 2, + "{\"stage\":\"persist-replacement\"}", + "persisted", + 0, + None, + ), + ), + "persist replacement", + ) + .await?; + + connection + .execute( + ENQUEUE_OTS_TRAJECTORY_SQL, + params![ + identity.queued_id.clone(), + identity.tenant.clone(), + identity.agent_id.clone(), + "queue-session-a".to_string(), + "queue-outcome-a".to_string(), + 2_i64, + "{\"stage\":\"enqueue-insert\"}".to_string(), + ], + ) + .await + .map_err(|error| schema_query_error("probe OTS enqueue insert", error))?; + require_ots_probe_state( + connection, + &identity.queued_id, + expected_ots_probe_state( + &identity, + ( + "queue-session-a", + "queue-outcome-a", + 2, + "{\"stage\":\"enqueue-insert\"}", + "queued", + 0, + None, + ), + ), + "enqueue insert", + ) + .await?; + + connection + .execute( + ENQUEUE_OTS_TRAJECTORY_SQL, + params![ + identity.queued_id.clone(), + identity.tenant.clone(), + identity.agent_id.clone(), + "queue-session-b".to_string(), + "queue-outcome-b".to_string(), + 3_i64, + "{\"stage\":\"enqueue-conflict\"}".to_string(), + ], + ) + .await + .map_err(|error| schema_query_error("probe OTS enqueue conflict update", error))?; + require_ots_probe_state( + connection, + &identity.queued_id, + expected_ots_probe_state( + &identity, + ( + "queue-session-b", + "queue-outcome-b", + 3, + "{\"stage\":\"enqueue-conflict\"}", + "queued", + 0, + None, + ), + ), + "enqueue conflict update", + ) + .await?; + + connection + .execute( + MARK_OTS_TRAJECTORY_FAILED_SQL, + params![identity.queued_id.clone(), OTS_PROBE_FAILURE.to_string()], + ) + .await + .map_err(|error| schema_query_error("probe OTS failed status update", error))?; + require_ots_probe_state( + connection, + &identity.queued_id, + expected_ots_probe_state( + &identity, + ( + "queue-session-b", + "queue-outcome-b", + 3, + "{\"stage\":\"enqueue-conflict\"}", + "failed", + 1, + Some(OTS_PROBE_FAILURE), + ), + ), + "failed status update", + ) + .await?; + + connection + .execute( + MARK_OTS_TRAJECTORY_PERSISTED_SQL, + params![identity.queued_id.clone()], + ) + .await + .map_err(|error| schema_query_error("probe OTS persisted status update", error))?; + require_ots_probe_state( + connection, + &identity.queued_id, + expected_ots_probe_state( + &identity, + ( + "queue-session-b", + "queue-outcome-b", + 3, + "{\"stage\":\"enqueue-conflict\"}", + "persisted", + 1, + None, + ), + ), + "persisted status update", + ) + .await?; + Ok(()) +} + +fn expected_ots_probe_state( + identity: &OtsProbeIdentity, + expected: ExpectedOtsProbeState<'_>, +) -> OtsProbeState { + let (session_id, outcome, turn_count, data, persistence_status, persist_attempts, last_error) = + expected; + OtsProbeState { + tenant: identity.tenant.clone(), + agent_id: identity.agent_id.clone(), + session_id: session_id.to_string(), + outcome: outcome.to_string(), + turn_count, + data: data.to_string(), + persistence_status: persistence_status.to_string(), + persist_attempts, + last_error: last_error.map(str::to_string), + } +} + +async fn require_ots_probe_state( + connection: &Connection, + trajectory_id: &str, + expected: OtsProbeState, + stage: &str, +) -> Result<(), PersistenceError> { + let mut rows = connection + .query( + "SELECT tenant, agent_id, session_id, outcome, turn_count, data, + persistence_status, persist_attempts, last_error + FROM ots_trajectories WHERE trajectory_id = ?1", + [trajectory_id], + ) + .await + .map_err(|error| schema_query_error("inspect OTS trigger probe state", error))?; + let row = rows + .next() + .await + .map_err(|error| schema_query_error("read OTS trigger probe state", error))? + .ok_or_else(|| { + compatibility_error(format!("OTS trigger probe row is missing after {stage}")) + })?; + let actual = OtsProbeState { + tenant: row + .get::(0) + .map_err(|error| schema_query_error("decode OTS probe tenant", error))?, + agent_id: row + .get::(1) + .map_err(|error| schema_query_error("decode OTS probe agent", error))?, + session_id: row + .get::(2) + .map_err(|error| schema_query_error("decode OTS probe session", error))?, + outcome: row + .get::(3) + .map_err(|error| schema_query_error("decode OTS probe outcome", error))?, + turn_count: row + .get::(4) + .map_err(|error| schema_query_error("decode OTS probe turn count", error))?, + data: row + .get::(5) + .map_err(|error| schema_query_error("decode OTS probe data", error))?, + persistence_status: row + .get::(6) + .map_err(|error| schema_query_error("decode OTS probe status", error))?, + persist_attempts: row + .get::(7) + .map_err(|error| schema_query_error("decode OTS probe attempts", error))?, + last_error: row + .get::>(8) + .map_err(|error| schema_query_error("decode OTS probe error", error))?, + }; + if actual != expected { + return Err(compatibility_error(format!( + "OTS trigger probe produced incompatible state after {stage}: expected {expected:?}, found {actual:?}" + ))); + } + Ok(()) +} + +fn schema_query_error(context: &str, error: libsql::Error) -> PersistenceError { + PersistenceError::Storage(format!( + "Turso schema introspection failed while attempting to {context}: {error} ({error:?})" + )) +} diff --git a/crates/temper-store-turso/src/migrations/schema_ots_trigger.rs b/crates/temper-store-turso/src/migrations/schema_ots_trigger.rs new file mode 100644 index 000000000..90d0e1deb --- /dev/null +++ b/crates/temper-store-turso/src/migrations/schema_ots_trigger.rs @@ -0,0 +1,283 @@ +use libsql::Connection; +use temper_runtime::persistence::PersistenceError; + +use super::schema_snapshot::{ + IndexColumn, UniqueKeyCapability, compatibility_error, index_capability, table_capability, +}; +use super::schema_sql::{canonical_tokens, normalize_schema_ddl}; +use super::schema_trigger::{TriggerCapability, capture_triggers}; + +const AUDIT_COLUMN: &str = "trajectory_id"; + +struct AuditSink { + table: String, +} + +pub(super) async fn validate_ots_audit_trigger_contracts( + connection: &Connection, + triggers: &[(&str, &TriggerCapability)], +) -> Result<(), PersistenceError> { + for (name, trigger) in triggers { + let sink = parse_audit_sink(name, trigger).ok_or_else(|| { + compatibility_error(format!( + "table 'ots_trajectories' has unsupported executable trigger extension \ + '{name}'; legacy extensions must be an unconditional AFTER INSERT audit \ + trigger with exactly one INSERT INTO (trajectory_id) \ + VALUES (NEW.trajectory_id) statement" + )) + })?; + validate_audit_sink(connection, name, &sink).await?; + } + Ok(()) +} + +fn parse_audit_sink(name: &str, trigger: &TriggerCapability) -> Option { + if trigger.table != "ots_trajectories" { + return None; + } + let tokens = canonical_tokens(&trigger.definition); + let expected_name = name.to_ascii_lowercase(); + if tokens.len() != 22 + || tokens[0] != "create" + || tokens[1] != "trigger" + || tokens[2] != expected_name + || tokens[3] != "after" + || tokens[4] != "insert" + || tokens[5] != "on" + || tokens[6] != "ots_trajectories" + || tokens[7] != "begin" + || tokens[8] != "insert" + || tokens[9] != "into" + || !is_plain_identifier(&tokens[10]) + || tokens[11] != "(" + || tokens[12] != AUDIT_COLUMN + || tokens[13] != ")" + || tokens[14] != "values" + || tokens[15] != "(" + || tokens[16] != "new" + || tokens[17] != "." + || tokens[18] != AUDIT_COLUMN + || tokens[19] != ")" + || tokens[20] != ";" + || tokens[21] != "end" + { + return None; + } + Some(AuditSink { + table: tokens[10].clone(), + }) +} + +async fn validate_audit_sink( + connection: &Connection, + trigger_name: &str, + sink: &AuditSink, +) -> Result<(), PersistenceError> { + let (actual_name, definition) = audit_table_definition(connection, &sink.table).await?; + if !normalize_schema_ddl(&definition).starts_with("create table ") { + return Err(unsupported_sink( + trigger_name, + &actual_name, + "the sink is not a plain table", + )); + } + + let table = table_capability(connection, &actual_name).await?; + let column = table + .columns + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case(AUDIT_COLUMN)); + if table.columns.len() != 1 + || column.is_none_or(|(_, column)| { + column.affinity != "TEXT" || column.default.is_some() || column.hidden != 0 + }) + { + return Err(unsupported_sink( + trigger_name, + &actual_name, + "the sink must contain only one visible TEXT trajectory_id column without a default", + )); + } + if !table.foreign_keys.is_empty() || !table.restricted_semantics.is_empty() { + return Err(unsupported_sink( + trigger_name, + &actual_name, + "the sink must not execute foreign keys, checks, generated columns, collations, or other table restrictions", + )); + } + if !table.unique_keys.iter().all(is_safe_trajectory_key) { + return Err(unsupported_sink( + trigger_name, + &actual_name, + "the sink has an unsafe unique-key definition", + )); + } + if !capture_triggers(connection, Some(&actual_name)) + .await? + .is_empty() + { + return Err(unsupported_sink( + trigger_name, + &actual_name, + "the sink has executable triggers", + )); + } + validate_audit_indexes(connection, trigger_name, &actual_name).await +} + +async fn validate_audit_indexes( + connection: &Connection, + trigger_name: &str, + table: &str, +) -> Result<(), PersistenceError> { + let mut rows = connection + .query( + "SELECT name FROM sqlite_schema + WHERE type = 'index' AND sql IS NOT NULL + AND name NOT GLOB 'sqlite_*' AND tbl_name COLLATE NOCASE = ?1 + ORDER BY name", + [table], + ) + .await + .map_err(|error| sink_query_error("list audit sink indexes", error))?; + let mut names = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|error| sink_query_error("read audit sink index", error))? + { + names.push( + row.get::(0) + .map_err(|error| sink_query_error("decode audit sink index name", error))?, + ); + } + drop(rows); + + for name in names { + let index = index_capability(connection, &name).await?; + if index.partial + || index.predicate.is_some() + || index.columns.len() != 1 + || !is_safe_trajectory_column(&index.columns[0]) + { + return Err(unsupported_sink( + trigger_name, + table, + &format!("the sink has executable or non-canonical index '{name}'"), + )); + } + } + Ok(()) +} + +async fn audit_table_definition( + connection: &Connection, + table: &str, +) -> Result<(String, String), PersistenceError> { + let mut rows = connection + .query( + "SELECT name, sql FROM sqlite_schema + WHERE type = 'table' AND name COLLATE NOCASE = ?1 + ORDER BY name LIMIT 1", + [table], + ) + .await + .map_err(|error| sink_query_error("inspect audit sink table", error))?; + let row = rows + .next() + .await + .map_err(|error| sink_query_error("read audit sink table", error))? + .ok_or_else(|| { + compatibility_error(format!( + "OTS audit trigger references missing table '{table}'" + )) + })?; + Ok(( + row.get::(0) + .map_err(|error| sink_query_error("decode audit sink table name", error))?, + row.get::(1) + .map_err(|error| sink_query_error("decode audit sink table definition", error))?, + )) +} + +fn is_safe_trajectory_key(key: &UniqueKeyCapability) -> bool { + !key.partial + && key.predicate.is_none() + && key.columns.len() == 1 + && is_safe_trajectory_column(&key.columns[0]) +} + +fn is_safe_trajectory_column(column: &IndexColumn) -> bool { + column + .name + .as_deref() + .is_some_and(|name| name.eq_ignore_ascii_case(AUDIT_COLUMN)) + && !column.descending + && matches!(column.collation.as_deref(), None | Some("binary")) +} + +fn is_plain_identifier(value: &str) -> bool { + !value.is_empty() + && !value.starts_with("sqlite_") + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'$')) +} + +fn unsupported_sink(trigger: &str, table: &str, reason: &str) -> PersistenceError { + compatibility_error(format!( + "table 'ots_trajectories' trigger extension '{trigger}' has unsupported audit sink \ + '{table}': {reason}" + )) +} + +fn sink_query_error(context: &str, error: libsql::Error) -> PersistenceError { + PersistenceError::Storage(format!( + "Turso schema introspection failed while attempting to {context}: {error} ({error:?})" + )) +} + +#[cfg(test)] +mod tests { + use super::parse_audit_sink; + use crate::migrations::schema_sql::normalize_schema_ddl; + use crate::migrations::schema_trigger::TriggerCapability; + + fn trigger(definition: &str) -> TriggerCapability { + TriggerCapability { + table: "ots_trajectories".into(), + definition: normalize_schema_ddl(definition), + } + } + + #[test] + fn parses_exact_unconditional_audit_contract() { + let capability = trigger( + "CREATE TRIGGER audit_ots_insert AFTER INSERT ON ots_trajectories + BEGIN + INSERT INTO ots_audit (trajectory_id) VALUES (NEW.trajectory_id); + END", + ); + let sink = parse_audit_sink("audit_ots_insert", &capability).expect("audit contract"); + assert_eq!(sink.table, "ots_audit"); + } + + #[test] + fn rejects_conditions_and_additional_statements() { + let conditional = trigger( + "CREATE TRIGGER audit_ots_insert AFTER INSERT ON ots_trajectories + WHEN NEW.tenant = 'probe' + BEGIN INSERT INTO ots_audit (trajectory_id) VALUES (NEW.trajectory_id); END", + ); + assert!(parse_audit_sink("audit_ots_insert", &conditional).is_none()); + + let mutating = trigger( + "CREATE TRIGGER audit_ots_insert AFTER INSERT ON ots_trajectories + BEGIN + INSERT INTO ots_audit (trajectory_id) VALUES (NEW.trajectory_id); + UPDATE ots_trajectories SET entity_type = 'changed'; + END", + ); + assert!(parse_audit_sink("audit_ots_insert", &mutating).is_none()); + } +} diff --git a/crates/temper-store-turso/src/migrations/schema_snapshot.rs b/crates/temper-store-turso/src/migrations/schema_snapshot.rs new file mode 100644 index 000000000..276943225 --- /dev/null +++ b/crates/temper-store-turso/src/migrations/schema_snapshot.rs @@ -0,0 +1,489 @@ +use std::collections::{BTreeMap, BTreeSet}; + +use libsql::Connection; +use temper_runtime::persistence::PersistenceError; + +use super::schema_sql::{predicate_after_where, restricted_table_semantics}; +use super::schema_trigger::{TriggerCapability, capture_triggers}; +pub(super) use super::schema_verify::verify_schema; + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub(super) struct ColumnCapability { + pub affinity: String, + pub not_null: bool, + pub default: Option, + pub primary_key_position: i64, + pub hidden: i64, +} + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub(super) struct IndexColumn { + pub name: Option, + pub descending: bool, + pub collation: Option, +} + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub(super) struct ForeignKeyPart { + pub id: i64, + pub sequence: i64, + pub target_table: String, + pub source_column: String, + pub target_column: Option, + pub on_update: String, + pub on_delete: String, + pub match_kind: String, +} + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub(super) struct UniqueKeyCapability { + pub partial: bool, + pub columns: Vec, + pub predicate: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct TableCapability { + pub columns: BTreeMap, + pub unique_keys: BTreeSet, + pub foreign_keys: BTreeSet, + pub restricted_semantics: BTreeSet, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct IndexCapability { + pub table: String, + pub unique: bool, + pub partial: bool, + pub columns: Vec, + pub predicate: Option, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct SchemaSnapshot { + pub tables: BTreeMap, + pub indexes: BTreeMap, + pub triggers: BTreeMap, +} + +impl SchemaSnapshot { + pub(super) fn manifest(&self) -> String { + super::schema_manifest::canonical_manifest(self) + } +} + +pub(super) async fn capture_schema( + connection: &Connection, +) -> Result { + let table_names = object_names(connection, "table").await?; + let index_names = named_index_names(connection).await?; + + let mut tables = BTreeMap::new(); + for name in table_names { + tables.insert(name.clone(), table_capability(connection, &name).await?); + } + + let mut indexes = BTreeMap::new(); + for name in index_names { + indexes.insert(name.clone(), index_capability(connection, &name).await?); + } + + Ok(SchemaSnapshot { + tables, + indexes, + triggers: capture_triggers(connection, None).await?, + }) +} + +async fn object_names( + connection: &Connection, + object_type: &str, +) -> Result, PersistenceError> { + let mut rows = connection + .query( + "SELECT name FROM sqlite_schema + WHERE type = ?1 AND name NOT GLOB 'sqlite_*' + ORDER BY name", + [object_type], + ) + .await + .map_err(|error| query_error("list schema objects", error))?; + let mut names = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|error| query_error("read schema object", error))? + { + names.push( + row.get::(0) + .map_err(|error| query_error("decode schema object name", error))?, + ); + } + Ok(names) +} + +async fn named_index_names(connection: &Connection) -> Result, PersistenceError> { + let mut rows = connection + .query( + "SELECT name FROM sqlite_schema + WHERE type = 'index' AND sql IS NOT NULL AND name NOT GLOB 'sqlite_*' + ORDER BY name", + (), + ) + .await + .map_err(|error| query_error("list named indexes", error))?; + let mut names = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|error| query_error("read named index", error))? + { + names.push( + row.get::(0) + .map_err(|error| query_error("decode index name", error))?, + ); + } + Ok(names) +} + +pub(super) async fn object_kind( + connection: &Connection, + name: &str, +) -> Result, PersistenceError> { + let mut rows = connection + .query( + "SELECT type FROM sqlite_schema WHERE name = ?1 ORDER BY type LIMIT 1", + [name], + ) + .await + .map_err(|error| query_error("inspect schema object kind", error))?; + let row = rows + .next() + .await + .map_err(|error| query_error("read schema object kind", error))?; + row.map(|row| { + row.get::(0) + .map_err(|error| query_error("decode schema object kind", error)) + }) + .transpose() +} + +pub(super) async fn table_capability( + connection: &Connection, + table: &str, +) -> Result { + let pragma = format!("PRAGMA table_xinfo({})", quote_identifier(table)); + let mut rows = connection + .query(&pragma, ()) + .await + .map_err(|error| query_error("inspect table columns", error))?; + let mut columns = BTreeMap::new(); + while let Some(row) = rows + .next() + .await + .map_err(|error| query_error("read table column", error))? + { + let name = row + .get::(1) + .map_err(|error| query_error("decode column name", error))?; + let declared_type = row + .get::(2) + .map_err(|error| query_error("decode column type", error))?; + let default = row + .get::>(4) + .map_err(|error| query_error("decode column default", error))?; + columns.insert( + name, + ColumnCapability { + affinity: type_affinity(&declared_type).to_string(), + not_null: row + .get::(3) + .map_err(|error| query_error("decode not-null flag", error))? + != 0, + default: default.map(|value| normalize_default(&value)), + primary_key_position: row + .get::(5) + .map_err(|error| query_error("decode primary-key position", error))?, + hidden: row + .get::(6) + .map_err(|error| query_error("decode hidden-column flag", error))?, + }, + ); + } + drop(rows); + + Ok(TableCapability { + columns, + unique_keys: unique_keys(connection, table).await?, + foreign_keys: foreign_keys(connection, table).await?, + restricted_semantics: restricted_table_semantics( + &table_definition(connection, table).await?, + ), + }) +} + +async fn table_definition( + connection: &Connection, + table: &str, +) -> Result { + let mut rows = connection + .query( + "SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = ?1", + [table], + ) + .await + .map_err(|error| query_error("inspect table definition", error))?; + rows.next() + .await + .map_err(|error| query_error("read table definition", error))? + .ok_or_else(|| compatibility_error(format!("required table '{table}' is missing")))? + .get::(0) + .map_err(|error| query_error("decode table definition", error)) +} + +async fn unique_keys( + connection: &Connection, + table: &str, +) -> Result, PersistenceError> { + let pragma = format!("PRAGMA index_list({})", quote_identifier(table)); + let mut rows = connection + .query(&pragma, ()) + .await + .map_err(|error| query_error("inspect table indexes", error))?; + let mut indexes = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|error| query_error("read table index", error))? + { + let unique = row + .get::(2) + .map_err(|error| query_error("decode unique-index flag", error))? + != 0; + if unique { + indexes.push(( + row.get::(1) + .map_err(|error| query_error("decode unique-index name", error))?, + row.get::(4) + .map_err(|error| query_error("decode partial-index flag", error))? + != 0, + )); + } + } + drop(rows); + + let mut keys = BTreeSet::new(); + for (name, partial) in indexes { + let definition = index_definition(connection, &name).await?; + keys.insert(UniqueKeyCapability { + partial, + columns: index_columns(connection, &name).await?, + predicate: definition.as_deref().and_then(predicate_after_where), + }); + } + Ok(keys) +} + +async fn index_definition( + connection: &Connection, + index: &str, +) -> Result, PersistenceError> { + let mut rows = connection + .query( + "SELECT sql FROM sqlite_schema WHERE type = 'index' AND name = ?1", + [index], + ) + .await + .map_err(|error| query_error("inspect unique-index definition", error))?; + rows.next() + .await + .map_err(|error| query_error("read unique-index definition", error))? + .ok_or_else(|| compatibility_error(format!("unique index '{index}' is missing")))? + .get::>(0) + .map_err(|error| query_error("decode unique-index definition", error)) +} + +async fn foreign_keys( + connection: &Connection, + table: &str, +) -> Result, PersistenceError> { + let pragma = format!("PRAGMA foreign_key_list({})", quote_identifier(table)); + let mut rows = connection + .query(&pragma, ()) + .await + .map_err(|error| query_error("inspect foreign keys", error))?; + let mut keys = BTreeSet::new(); + while let Some(row) = rows + .next() + .await + .map_err(|error| query_error("read foreign key", error))? + { + keys.insert(ForeignKeyPart { + id: row + .get::(0) + .map_err(|error| query_error("decode foreign-key id", error))?, + sequence: row + .get::(1) + .map_err(|error| query_error("decode foreign-key sequence", error))?, + target_table: row + .get::(2) + .map_err(|error| query_error("decode foreign-key table", error))?, + source_column: row + .get::(3) + .map_err(|error| query_error("decode foreign-key source", error))?, + target_column: row + .get::>(4) + .map_err(|error| query_error("decode foreign-key target", error))?, + on_update: row + .get::(5) + .map_err(|error| query_error("decode foreign-key update action", error))?, + on_delete: row + .get::(6) + .map_err(|error| query_error("decode foreign-key delete action", error))?, + match_kind: row + .get::(7) + .map_err(|error| query_error("decode foreign-key match", error))?, + }); + } + Ok(keys) +} + +pub(super) async fn index_capability( + connection: &Connection, + index: &str, +) -> Result { + let mut schema_rows = connection + .query( + "SELECT tbl_name, sql FROM sqlite_schema WHERE type = 'index' AND name = ?1", + [index], + ) + .await + .map_err(|error| query_error("inspect named index", error))?; + let schema_row = schema_rows + .next() + .await + .map_err(|error| query_error("read named index", error))? + .ok_or_else(|| compatibility_error(format!("required index '{index}' is missing")))?; + let table = schema_row + .get::(0) + .map_err(|error| query_error("decode index owner", error))?; + let sql = schema_row + .get::(1) + .map_err(|error| query_error("decode index SQL", error))?; + drop(schema_rows); + + let pragma = format!("PRAGMA index_list({})", quote_identifier(&table)); + let mut rows = connection + .query(&pragma, ()) + .await + .map_err(|error| query_error("inspect index semantics", error))?; + let mut flags = None; + while let Some(row) = rows + .next() + .await + .map_err(|error| query_error("read index semantics", error))? + { + let name = row + .get::(1) + .map_err(|error| query_error("decode index name", error))?; + if name == index { + flags = Some(( + row.get::(2) + .map_err(|error| query_error("decode unique flag", error))? + != 0, + row.get::(4) + .map_err(|error| query_error("decode partial flag", error))? + != 0, + )); + break; + } + } + let (unique, partial) = flags.ok_or_else(|| { + compatibility_error(format!("index '{index}' is not owned by table '{table}'")) + })?; + + Ok(IndexCapability { + table, + unique, + partial, + columns: index_columns(connection, index).await?, + predicate: predicate_after_where(&sql), + }) +} + +pub(super) async fn index_columns( + connection: &Connection, + index: &str, +) -> Result, PersistenceError> { + let pragma = format!("PRAGMA index_xinfo({})", quote_identifier(index)); + let mut rows = connection + .query(&pragma, ()) + .await + .map_err(|error| query_error("inspect index columns", error))?; + let mut columns = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|error| query_error("read index column", error))? + { + let key = row + .get::(5) + .map_err(|error| query_error("decode index key flag", error))?; + if key == 0 { + continue; + } + columns.push(IndexColumn { + name: row + .get::>(2) + .map_err(|error| query_error("decode index column name", error))?, + descending: row + .get::(3) + .map_err(|error| query_error("decode index sort order", error))? + != 0, + collation: row + .get::>(4) + .map_err(|error| query_error("decode index collation", error))? + .map(|value| value.to_ascii_lowercase()), + }); + } + Ok(columns) +} + +pub(super) fn type_affinity(declared_type: &str) -> &'static str { + let upper = declared_type.to_ascii_uppercase(); + if upper.contains("INT") { + "INTEGER" + } else if upper.contains("CHAR") || upper.contains("CLOB") || upper.contains("TEXT") { + "TEXT" + } else if upper.contains("BLOB") || upper.is_empty() { + "BLOB" + } else if upper.contains("REAL") || upper.contains("FLOA") || upper.contains("DOUB") { + "REAL" + } else { + "NUMERIC" + } +} + +pub(super) fn normalize_default(value: &str) -> String { + let mut normalized = value.trim(); + while normalized.starts_with('(') && normalized.ends_with(')') { + normalized = normalized[1..normalized.len() - 1].trim(); + } + normalized.split_whitespace().collect::>().join(" ") +} + +fn quote_identifier(identifier: &str) -> String { + format!("\"{}\"", identifier.replace('"', "\"\"")) +} + +fn query_error(context: &str, error: libsql::Error) -> PersistenceError { + PersistenceError::Storage(format!( + "Turso schema introspection failed while attempting to {context}: {error} ({error:?})" + )) +} + +pub(super) fn compatibility_error(message: String) -> PersistenceError { + PersistenceError::Storage(format!( + "Turso schema compatibility check failed: {message}" + )) +} diff --git a/crates/temper-store-turso/src/migrations/schema_sql.rs b/crates/temper-store-turso/src/migrations/schema_sql.rs new file mode 100644 index 000000000..216c1522a --- /dev/null +++ b/crates/temper-store-turso/src/migrations/schema_sql.rs @@ -0,0 +1,265 @@ +use std::collections::BTreeSet; + +pub(super) const RESTRICTED_TABLE_SEQUENCES: &[(&str, &[&str])] = &[ + ("AUTOINCREMENT", &["AUTOINCREMENT"]), + ("CHECK", &["CHECK"]), + ("COLLATE", &["COLLATE"]), + ("DEFERRABLE", &["DEFERRABLE"]), + ("GENERATED", &["GENERATED"]), + ("INITIALLY", &["INITIALLY"]), + ("ON CONFLICT", &["ON", "CONFLICT"]), + ("STRICT", &["STRICT"]), + ("WITHOUT ROWID", &["WITHOUT", "ROWID"]), +]; + +pub(super) fn contains_sequence(sql: &str, sequences: &[&[&str]]) -> Option { + let tokens = tokens(sql); + sequences + .iter() + .position(|sequence| contains_tokens(&tokens, sequence)) +} + +pub(super) fn restricted_table_semantics(sql: &str) -> BTreeSet { + let tokens = tokens(sql); + RESTRICTED_TABLE_SEQUENCES + .iter() + .filter(|(_, sequence)| contains_tokens(&tokens, sequence)) + .map(|(name, _)| (*name).to_string()) + .collect() +} + +pub(super) fn predicate_after_where(sql: &str) -> Option { + token_spans(sql) + .into_iter() + .find(|token| token.value == "WHERE") + .map(|token| normalize_sql_fragment(&sql[token.end..])) +} + +pub(super) fn normalize_schema_ddl(sql: &str) -> String { + let mut tokens = canonical_tokens(sql); + if tokens.last().is_some_and(|token| token == ";") { + tokens.pop(); + } + if tokens.len() >= 5 + && tokens[0] == "create" + && tokens[1] == "table" + && tokens[2] == "if" + && tokens[3] == "not" + && tokens[4] == "exists" + { + tokens.drain(2..5); + } + tokens.join(" ") +} + +fn normalize_sql_fragment(sql: &str) -> String { + let mut tokens = canonical_tokens(sql); + if tokens.last().is_some_and(|token| token == ";") { + tokens.pop(); + } + tokens.join(" ") +} + +pub(super) fn canonical_tokens(sql: &str) -> Vec { + let bytes = sql.as_bytes(); + let mut tokens = Vec::new(); + let mut cursor = 0; + while cursor < bytes.len() { + match bytes[cursor] { + b'\'' | b'"' | b'`' => { + let end = skip_quoted(bytes, cursor, bytes[cursor]); + tokens.push(sql[cursor..end].to_string()); + cursor = end; + } + b'[' => { + let end = skip_quoted(bytes, cursor, b']'); + tokens.push(sql[cursor..end].to_string()); + cursor = end; + } + b'-' if bytes.get(cursor + 1) == Some(&b'-') => { + cursor += 2; + while cursor < bytes.len() && bytes[cursor] != b'\n' { + cursor += 1; + } + } + b'/' if bytes.get(cursor + 1) == Some(&b'*') => { + cursor += 2; + while cursor + 1 < bytes.len() + && !(bytes[cursor] == b'*' && bytes[cursor + 1] == b'/') + { + cursor += 1; + } + cursor = (cursor + 2).min(bytes.len()); + } + byte if byte.is_ascii_whitespace() => cursor += 1, + byte if is_identifier_byte(byte) => { + let start = cursor; + cursor += 1; + while cursor < bytes.len() && is_identifier_byte(bytes[cursor]) { + cursor += 1; + } + tokens.push(sql[start..cursor].to_ascii_lowercase()); + } + byte => { + tokens.push((byte as char).to_string()); + cursor += 1; + } + } + } + tokens +} + +fn contains_tokens(tokens: &[String], sequence: &[&str]) -> bool { + tokens.windows(sequence.len()).any(|window| { + window + .iter() + .zip(sequence) + .all(|(left, right)| left == right) + }) +} + +fn tokens(sql: &str) -> Vec { + token_spans(sql) + .into_iter() + .map(|token| token.value) + .collect() +} + +struct TokenSpan { + value: String, + end: usize, +} + +fn token_spans(sql: &str) -> Vec { + let bytes = sql.as_bytes(); + let mut tokens = Vec::new(); + let mut cursor = 0; + while cursor < bytes.len() { + match bytes[cursor] { + b'\'' | b'"' | b'`' => cursor = skip_quoted(bytes, cursor, bytes[cursor]), + b'[' => cursor = skip_quoted(bytes, cursor, b']'), + b'-' if bytes.get(cursor + 1) == Some(&b'-') => { + cursor += 2; + while cursor < bytes.len() && bytes[cursor] != b'\n' { + cursor += 1; + } + } + b'/' if bytes.get(cursor + 1) == Some(&b'*') => { + cursor += 2; + while cursor + 1 < bytes.len() + && !(bytes[cursor] == b'*' && bytes[cursor + 1] == b'/') + { + cursor += 1; + } + cursor = (cursor + 2).min(bytes.len()); + } + byte if is_identifier_byte(byte) => { + let start = cursor; + cursor += 1; + while cursor < bytes.len() && is_identifier_byte(bytes[cursor]) { + cursor += 1; + } + tokens.push(TokenSpan { + value: sql[start..cursor].to_ascii_uppercase(), + end: cursor, + }); + } + _ => cursor += 1, + } + } + tokens +} + +fn skip_quoted(bytes: &[u8], mut cursor: usize, terminator: u8) -> usize { + cursor += 1; + while cursor < bytes.len() { + if bytes[cursor] != terminator { + cursor += 1; + continue; + } + if bytes.get(cursor + 1) == Some(&terminator) { + cursor += 2; + continue; + } + return cursor + 1; + } + cursor +} + +fn is_identifier_byte(byte: u8) -> bool { + byte.is_ascii_alphanumeric() || matches!(byte, b'_' | b'$') || !byte.is_ascii() +} + +#[cfg(test)] +mod tests { + use super::{ + contains_sequence, normalize_schema_ddl, predicate_after_where, restricted_table_semantics, + }; + + #[test] + fn token_matching_handles_punctuation_and_ignores_quoted_text() { + let sequences: &[&[&str]] = &[&["CHECK"], &["WITHOUT", "ROWID"]]; + assert_eq!( + contains_sequence("data TEXT,CHECK(length(data)>0)", sequences), + Some(0) + ); + assert_eq!( + contains_sequence("value TEXT)WITHOUT/* gap */ROWID", sequences), + Some(1) + ); + assert_eq!( + contains_sequence("value TEXT DEFAULT 'CHECK WITHOUT ROWID'", sequences), + None + ); + assert_eq!(contains_sequence("\"CHECK\" TEXT", sequences), None); + } + + #[test] + fn restricted_semantics_are_ordered_and_ignore_quoted_text() { + assert_eq!( + restricted_table_semantics( + "CREATE TABLE sample ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + note TEXT DEFAULT 'CHECK STRICT', + CHECK(length(note) > 0) + ) STRICT" + ) + .into_iter() + .collect::>(), + vec!["AUTOINCREMENT", "CHECK", "STRICT"] + ); + } + + #[test] + fn partial_predicate_handles_compact_syntax_and_quoted_where() { + assert_eq!( + predicate_after_where( + "CREATE INDEX sample ON records(json_extract(value, '$.where'))WHERE(length(id)>0);" + ) + .as_deref(), + Some("( length ( id ) > 0 )") + ); + } + + #[test] + fn schema_ddl_normalization_ignores_formatting_but_preserves_literals() { + assert_eq!( + normalize_schema_ddl( + "CREATE TABLE IF NOT EXISTS sample ( + value TEXT NOT NULL DEFAULT ('A B'), CHECK(length(value) > 0) + );" + ), + normalize_schema_ddl( + "create table sample(value text not null default('A B'),check(length(value)>0))" + ) + ); + assert_ne!( + normalize_schema_ddl("CREATE TABLE sample(value TEXT DEFAULT 'A B')"), + normalize_schema_ddl("CREATE TABLE sample(value TEXT DEFAULT 'a b')") + ); + assert_ne!( + normalize_schema_ddl("CREATE TABLE sample(id INTEGER PRIMARY KEY)"), + normalize_schema_ddl("CREATE TABLE sample(id INTEGER PRIMARYKEY)") + ); + } +} diff --git a/crates/temper-store-turso/src/migrations/schema_trigger.rs b/crates/temper-store-turso/src/migrations/schema_trigger.rs new file mode 100644 index 000000000..afc00e1fe --- /dev/null +++ b/crates/temper-store-turso/src/migrations/schema_trigger.rs @@ -0,0 +1,70 @@ +use std::collections::BTreeMap; + +use libsql::Connection; +use temper_runtime::persistence::PersistenceError; + +use super::schema_sql::normalize_schema_ddl; + +#[derive(Clone, Debug, Eq, PartialEq)] +pub(super) struct TriggerCapability { + pub table: String, + pub definition: String, +} + +pub(super) async fn capture_triggers( + connection: &Connection, + table: Option<&str>, +) -> Result, PersistenceError> { + let mut rows = if let Some(table) = table { + connection + .query( + "SELECT name, tbl_name, sql FROM sqlite_schema + WHERE type = 'trigger' AND name NOT GLOB 'sqlite_*' + AND tbl_name COLLATE NOCASE = ?1 + ORDER BY name", + [table], + ) + .await + } else { + connection + .query( + "SELECT name, tbl_name, sql FROM sqlite_schema + WHERE type = 'trigger' AND name NOT GLOB 'sqlite_*' + ORDER BY name", + (), + ) + .await + } + .map_err(|error| trigger_query_error("list triggers", error))?; + + let mut triggers = BTreeMap::new(); + while let Some(row) = rows + .next() + .await + .map_err(|error| trigger_query_error("read trigger", error))? + { + let name = row + .get::(0) + .map_err(|error| trigger_query_error("decode trigger name", error))?; + let owner = row + .get::(1) + .map_err(|error| trigger_query_error("decode trigger owner", error))?; + let definition = row + .get::(2) + .map_err(|error| trigger_query_error("decode trigger definition", error))?; + triggers.insert( + name, + TriggerCapability { + table: owner.to_ascii_lowercase(), + definition: normalize_schema_ddl(&definition), + }, + ); + } + Ok(triggers) +} + +fn trigger_query_error(context: &str, error: libsql::Error) -> PersistenceError { + PersistenceError::Storage(format!( + "Turso schema introspection failed while attempting to {context}: {error} ({error:?})" + )) +} diff --git a/crates/temper-store-turso/src/migrations/schema_verify.rs b/crates/temper-store-turso/src/migrations/schema_verify.rs new file mode 100644 index 000000000..f379daaa0 --- /dev/null +++ b/crates/temper-store-turso/src/migrations/schema_verify.rs @@ -0,0 +1,270 @@ +use libsql::Connection; +use temper_runtime::persistence::PersistenceError; + +use super::schema_ots_probe::validate_legacy_ots_triggers; +use super::schema_snapshot::{ + IndexCapability, SchemaSnapshot, TableCapability, compatibility_error, index_capability, + object_kind, table_capability, +}; +use super::schema_trigger::capture_triggers; + +pub(super) const EXTRA_COLUMN_POLICY: &str = + "allow-visible-nullable-no-default-non-primary-key-non-rowid-shadow-v2"; +pub(super) const EXTRA_INDEX_POLICY: &str = "allow-nonunique-full-plain-column-index-with-builtin-collation-and-sqlite-identifier-owners-v2"; +pub(super) const TRIGGER_POLICY: &str = concat!( + "exact-trigger-set-with-sqlite-identifier-owners-and-", + "parsed-audit-sink-contract-with-transaction-pinned-production-upsert-probe-v7" +); + +pub(super) async fn verify_schema( + connection: &Connection, + expected: &SchemaSnapshot, +) -> Result<(), PersistenceError> { + for (table_name, expected_table) in &expected.tables { + let kind = object_kind(connection, table_name).await?; + if kind.as_deref() != Some("table") { + return Err(compatibility_error(format!( + "capability '{table_name}' must be a table, found {}", + kind.as_deref().unwrap_or("no schema object") + ))); + } + let actual = table_capability(connection, table_name).await?; + verify_table(table_name, expected_table, &actual)?; + verify_triggers(connection, table_name, expected).await?; + } + + for (index_name, expected_index) in &expected.indexes { + verify_index(connection, index_name, expected_index).await?; + } + for table_name in expected.tables.keys() { + verify_index_extensions(connection, table_name, expected).await?; + } + Ok(()) +} + +async fn verify_triggers( + connection: &Connection, + table: &str, + expected: &SchemaSnapshot, +) -> Result<(), PersistenceError> { + let actual = capture_triggers(connection, Some(table)).await?; + let mut unexpected = Vec::new(); + for (name, actual_trigger) in &actual { + match expected.triggers.get(name) { + Some(expected_trigger) if expected_trigger == actual_trigger => {} + Some(expected_trigger) => { + return Err(compatibility_error(format!( + "trigger '{name}' has incompatible semantics: expected {expected_trigger:?}, found {actual_trigger:?}" + ))); + } + None => { + unexpected.push((name.as_str(), actual_trigger)); + } + } + } + if !unexpected.is_empty() { + if table == "ots_trajectories" { + validate_legacy_ots_triggers(connection, &unexpected).await?; + } else { + return Err(compatibility_error(format!( + "table '{table}' has unexpected executable trigger '{}'", + unexpected[0].0 + ))); + } + } + for (name, expected_trigger) in &expected.triggers { + if expected_trigger.table.eq_ignore_ascii_case(table) && !actual.contains_key(name) { + return Err(compatibility_error(format!( + "table '{table}' is missing required trigger '{name}'" + ))); + } + } + Ok(()) +} + +async fn verify_index_extensions( + connection: &Connection, + table: &str, + expected: &SchemaSnapshot, +) -> Result<(), PersistenceError> { + let mut rows = connection + .query( + "SELECT name FROM sqlite_schema + WHERE type = 'index' AND sql IS NOT NULL + AND name NOT GLOB 'sqlite_*' AND tbl_name COLLATE NOCASE = ?1 + ORDER BY name", + [table], + ) + .await + .map_err(|error| schema_query_error("list table indexes", error))?; + let mut names = Vec::new(); + while let Some(row) = rows + .next() + .await + .map_err(|error| schema_query_error("read table index", error))? + { + names.push( + row.get::(0) + .map_err(|error| schema_query_error("decode table index name", error))?, + ); + } + drop(rows); + + for name in names { + if expected.indexes.contains_key(&name) { + continue; + } + let actual = index_capability(connection, &name).await?; + if !is_safe_plain_index(table, &actual) { + return Err(compatibility_error(format!( + "table '{table}' has unexpected executable index extension '{name}' with semantics {actual:?}" + ))); + } + } + Ok(()) +} + +fn is_safe_plain_index(table: &str, index: &IndexCapability) -> bool { + index.table == table + && !index.unique + && !index.partial + && index.predicate.is_none() + && !index.columns.is_empty() + && index.columns.iter().all(|column| { + column.name.is_some() + && matches!( + column.collation.as_deref(), + None | Some("binary" | "nocase" | "rtrim") + ) + }) +} + +async fn verify_index( + connection: &Connection, + name: &str, + expected: &IndexCapability, +) -> Result<(), PersistenceError> { + let kind = object_kind(connection, name).await?; + if kind.as_deref() != Some("index") { + return Err(compatibility_error(format!( + "capability '{name}' must be an index, found {}", + kind.as_deref().unwrap_or("no schema object") + ))); + } + let actual = index_capability(connection, name).await?; + if &actual != expected { + return Err(compatibility_error(format!( + "index '{name}' has incompatible semantics: expected {expected:?}, found {actual:?}" + ))); + } + Ok(()) +} + +fn verify_table( + name: &str, + expected: &TableCapability, + actual: &TableCapability, +) -> Result<(), PersistenceError> { + for (column_name, expected_column) in &expected.columns { + let Some(actual_column) = actual.columns.get(column_name) else { + return Err(compatibility_error(format!( + "table '{name}' is missing required column '{column_name}'" + ))); + }; + if actual_column != expected_column { + return Err(compatibility_error(format!( + "table '{name}' column '{column_name}' has incompatible semantics: expected {expected_column:?}, found {actual_column:?}" + ))); + } + } + + for (column_name, column) in &actual.columns { + if expected.columns.contains_key(column_name) { + continue; + } + let shadows_rowid = matches!( + column_name.to_ascii_lowercase().as_str(), + "rowid" | "_rowid_" | "oid" + ); + if column.not_null + || column.default.is_some() + || column.primary_key_position != 0 + || column.hidden != 0 + || shadows_rowid + { + return Err(compatibility_error(format!( + "table '{name}' has unexpected required column or omission-unsafe extension '{column_name}' with semantics {column:?}" + ))); + } + } + + if actual.unique_keys != expected.unique_keys { + return Err(compatibility_error(format!( + "table '{name}' has incompatible unique key restrictions: expected {:?}, found {:?}", + expected.unique_keys, actual.unique_keys + ))); + } + if actual.foreign_keys != expected.foreign_keys { + return Err(compatibility_error(format!( + "table '{name}' has incompatible foreign key restrictions: expected {:?}, found {:?}", + expected.foreign_keys, actual.foreign_keys + ))); + } + if actual.restricted_semantics != expected.restricted_semantics { + return Err(compatibility_error(format!( + "table '{name}' has incompatible restricted table semantics: expected {:?}, found {:?}", + expected.restricted_semantics, actual.restricted_semantics + ))); + } + Ok(()) +} + +fn schema_query_error(context: &str, error: libsql::Error) -> PersistenceError { + PersistenceError::Storage(format!( + "Turso schema introspection failed while attempting to {context}: {error} ({error:?})" + )) +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use libsql::Builder; + + use super::{SchemaSnapshot, verify_index_extensions}; + + #[tokio::test] + async fn case_folded_index_owner_is_inventoried() { + let directory = tempfile::tempdir().expect("temporary database directory"); + let database = Builder::new_local(directory.path().join("case-folded-index-owner.db")) + .build() + .await + .expect("build temporary database"); + let connection = database.connect().expect("connect temporary database"); + connection + .execute("CREATE TABLE EVENTS(payload TEXT NOT NULL)", ()) + .await + .expect("create differently cased table owner"); + connection + .execute( + "CREATE INDEX events_case_folded_expression + ON EVENTS(json_extract(payload, 'invalid-path'))", + (), + ) + .await + .expect("create expression index with differently cased owner"); + let expected = SchemaSnapshot { + tables: BTreeMap::new(), + indexes: BTreeMap::new(), + triggers: BTreeMap::new(), + }; + + let error = verify_index_extensions(&connection, "events", &expected) + .await + .expect_err("SQLite-equivalent index owners must be inventoried"); + assert!( + error.to_string().contains("events_case_folded_expression"), + "{error}" + ); + } +} diff --git a/crates/temper-store-turso/src/migrations/schema_verify_tests.rs b/crates/temper-store-turso/src/migrations/schema_verify_tests.rs new file mode 100644 index 000000000..1470c8539 --- /dev/null +++ b/crates/temper-store-turso/src/migrations/schema_verify_tests.rs @@ -0,0 +1,361 @@ +use libsql::{Builder, Connection}; + +use super::catalog::MIGRATIONS; +use super::runner::migrate; + +mod ots_probe_replacement; +mod runtime_extensions; + +#[tokio::test] +async fn unexpected_required_column_prevents_ledgering_and_preserves_table() { + let (_directory, connection) = temporary_connection("required-column").await; + create_events(&connection, Some("must_fill TEXT NOT NULL"), None).await; + connection + .execute( + "INSERT INTO events ( + tenant, entity_type, entity_id, sequence_nr, event_type, payload, must_fill + ) VALUES ('tenant-a', 'Order', 'order-1', 1, 'Created', '{}', 'present')", + (), + ) + .await + .expect("seed restricted events table"); + let before = table_sql(&connection, "events").await; + + let error = migrate(&connection) + .await + .expect_err("an unexpected required column must prevent readiness"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains("migration 1"), "{diagnostic}"); + assert!(diagnostic.contains("must_fill"), "{diagnostic}"); + assert!( + diagnostic.contains("unexpected required column"), + "{diagnostic}" + ); + assert_eq!(table_sql(&connection, "events").await, before); + assert_eq!( + scalar_i64(&connection, "SELECT COUNT(*) FROM events").await, + 1 + ); + assert_eq!(ledger_count(&connection).await, 0); +} + +#[tokio::test] +async fn unexpected_unique_index_prevents_ledgering_without_dropping_index() { + let (_directory, connection) = temporary_connection("unique-index").await; + create_events(&connection, None, None).await; + connection + .execute( + "CREATE UNIQUE INDEX events_unexpected_unique + ON events(tenant, event_type)", + (), + ) + .await + .expect("unexpected unique index"); + + let error = migrate(&connection) + .await + .expect_err("an unexpected unique restriction must prevent readiness"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains("migration 1"), "{diagnostic}"); + assert!( + diagnostic.contains("unique key restrictions"), + "{diagnostic}" + ); + assert_eq!( + schema_kind(&connection, "events_unexpected_unique") + .await + .as_deref(), + Some("index") + ); + assert_eq!(ledger_count(&connection).await, 0); +} + +#[tokio::test] +async fn unexpected_foreign_key_prevents_ledgering_and_preserves_table() { + let (_directory, connection) = temporary_connection("foreign-key").await; + connection + .execute("CREATE TABLE tenant_guard (tenant TEXT PRIMARY KEY)", ()) + .await + .expect("foreign-key target"); + create_events( + &connection, + None, + Some("FOREIGN KEY (tenant) REFERENCES tenant_guard(tenant)"), + ) + .await; + let before = table_sql(&connection, "events").await; + + let error = migrate(&connection) + .await + .expect_err("an unexpected foreign-key restriction must prevent readiness"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains("migration 1"), "{diagnostic}"); + assert!( + diagnostic.contains("foreign key restrictions"), + "{diagnostic}" + ); + assert_eq!(table_sql(&connection, "events").await, before); + assert_eq!(ledger_count(&connection).await, 0); +} + +#[tokio::test] +async fn unexpected_check_semantics_prevent_ledgering_and_preserve_table() { + let (_directory, connection) = temporary_connection("check-semantics").await; + create_events(&connection, None, Some("CHECK (length(payload) > 0)")).await; + let before = table_sql(&connection, "events").await; + + let error = migrate(&connection) + .await + .expect_err("an unmodeled CHECK restriction must prevent readiness"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains("migration 1"), "{diagnostic}"); + assert!( + diagnostic.contains("restricted table semantics"), + "{diagnostic}" + ); + assert_eq!(table_sql(&connection, "events").await, before); + assert_eq!(ledger_count(&connection).await, 0); +} + +#[tokio::test] +async fn nullable_extension_remains_compatible_with_runtime_inserts() { + let (_directory, connection) = temporary_connection("nullable-extension").await; + create_events(&connection, Some("deployment_note TEXT"), None).await; + + migrate(&connection) + .await + .expect("a nullable extension must remain compatible"); + connection + .execute( + "INSERT INTO events ( + tenant, entity_type, entity_id, sequence_nr, event_type, payload + ) VALUES ('tenant-a', 'Order', 'order-1', 1, 'Created', '{}')", + (), + ) + .await + .expect("canonical runtime insert with nullable extension"); + assert_eq!( + scalar_i64(&connection, "SELECT COUNT(*) FROM events").await, + 1 + ); + assert_eq!(ledger_count(&connection).await, MIGRATIONS.len() as i64); +} + +#[tokio::test] +async fn nullable_extension_with_executable_default_prevents_ledgering() { + let (_directory, connection) = temporary_connection("unsafe-nullable-default").await; + create_events( + &connection, + Some("risky TEXT DEFAULT (json_extract('bad', '$'))"), + None, + ) + .await; + let before = table_sql(&connection, "events").await; + + let error = migrate(&connection) + .await + .expect_err("an executable default must not be treated as omission-safe"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains("migration 1"), "{diagnostic}"); + assert!(diagnostic.contains("risky"), "{diagnostic}"); + assert_eq!(table_sql(&connection, "events").await, before); + assert_eq!(ledger_count(&connection).await, 0); +} + +#[tokio::test] +async fn short_form_generated_column_prevents_ledgering() { + let (_directory, connection) = temporary_connection("short-generated").await; + create_events( + &connection, + Some("risky TEXT AS (json_extract(payload, '$.required')) NOT NULL"), + None, + ) + .await; + let before = table_sql(&connection, "events").await; + + let error = migrate(&connection) + .await + .expect_err("a short-form generated restriction must prevent readiness"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains("migration 1"), "{diagnostic}"); + assert!(diagnostic.contains("risky"), "{diagnostic}"); + assert!(diagnostic.contains("hidden: 2"), "{diagnostic}"); + assert_eq!(table_sql(&connection, "events").await, before); + assert_eq!(ledger_count(&connection).await, 0); +} + +#[tokio::test] +async fn partial_unique_index_cannot_replace_required_full_unique_key() { + let (_directory, connection) = temporary_connection("partial-unique").await; + create_events_with_identity_unique(&connection, None, None, false).await; + connection + .execute( + "CREATE UNIQUE INDEX events_partial_identity + ON events(tenant, entity_type, entity_id, sequence_nr) + WHERE length(tenant) > 0", + (), + ) + .await + .expect("partial identity index"); + + let error = migrate(&connection) + .await + .expect_err("a partial index must not satisfy a required full unique key"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains("migration 1"), "{diagnostic}"); + assert!( + diagnostic.contains("unique key restrictions"), + "{diagnostic}" + ); + assert_eq!( + schema_kind(&connection, "events_partial_identity") + .await + .as_deref(), + Some("index") + ); + assert_eq!(ledger_count(&connection).await, 0); +} + +#[tokio::test] +async fn generated_column_matching_add_step_reports_semantic_incompatibility() { + let (_directory, connection) = temporary_connection("generated-add-step").await; + create_events( + &connection, + Some("segment_index INTEGER AS (sequence_nr - 1)"), + None, + ) + .await; + let before = table_sql(&connection, "events").await; + + let error = migrate(&connection) + .await + .expect_err("a generated AddColumn target must prevent readiness"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains("migration 1"), "{diagnostic}"); + assert!(diagnostic.contains("segment_index"), "{diagnostic}"); + assert!(diagnostic.contains("hidden: 2"), "{diagnostic}"); + assert!(!diagnostic.contains("duplicate column"), "{diagnostic}"); + assert_eq!(table_sql(&connection, "events").await, before); + assert_eq!(ledger_count(&connection).await, 0); +} + +#[tokio::test] +async fn equivalent_lowercase_partial_predicate_remains_compatible() { + let (_directory, connection) = temporary_connection("lowercase-partial-predicate").await; + migrate(&connection).await.expect("install catalog"); + connection + .execute("DROP INDEX idx_blobs_expires_at", ()) + .await + .expect("drop canonical partial index"); + connection + .execute( + "CREATE INDEX idx_blobs_expires_at + ON blobs(expires_at) where expires_at is not null", + (), + ) + .await + .expect("create equivalent lowercase partial index"); + + migrate(&connection) + .await + .expect("keyword case must not make an equivalent predicate incompatible"); + assert_eq!(ledger_count(&connection).await, MIGRATIONS.len() as i64); +} + +async fn create_events( + connection: &Connection, + extra_column: Option<&str>, + extra_constraint: Option<&str>, +) { + create_events_with_identity_unique(connection, extra_column, extra_constraint, true).await; +} + +async fn create_events_with_identity_unique( + connection: &Connection, + extra_column: Option<&str>, + extra_constraint: Option<&str>, + include_identity_unique: bool, +) { + let mut definitions = vec![ + "id INTEGER PRIMARY KEY AUTOINCREMENT", + "tenant TEXT NOT NULL", + "entity_type TEXT NOT NULL", + "entity_id TEXT NOT NULL", + "sequence_nr INTEGER NOT NULL", + "event_type TEXT NOT NULL", + "payload TEXT NOT NULL", + "metadata TEXT", + "created_at TEXT NOT NULL DEFAULT (datetime('now'))", + ]; + if let Some(extra_column) = extra_column { + definitions.push(extra_column); + } + if include_identity_unique { + definitions.push("UNIQUE(tenant, entity_type, entity_id, sequence_nr)"); + } + if let Some(extra_constraint) = extra_constraint { + definitions.push(extra_constraint); + } + let sql = format!("CREATE TABLE events ({})", definitions.join(", ")); + connection + .execute(&sql, ()) + .await + .expect("create legacy events table"); +} + +async fn temporary_connection(label: &str) -> (tempfile::TempDir, Connection) { + let directory = tempfile::tempdir().expect("temporary database directory"); + let database = Builder::new_local(directory.path().join(format!("{label}.db"))) + .build() + .await + .expect("build temporary database"); + let connection = database.connect().expect("connect temporary database"); + (directory, connection) +} + +async fn table_sql(connection: &Connection, table: &str) -> String { + let mut rows = connection + .query( + "SELECT sql FROM sqlite_schema WHERE type = 'table' AND name = ?1", + [table], + ) + .await + .expect("query table SQL"); + rows.next() + .await + .expect("read table SQL") + .expect("table SQL row") + .get::(0) + .expect("decode table SQL") +} + +async fn schema_kind(connection: &Connection, name: &str) -> Option { + let mut rows = connection + .query( + "SELECT type FROM sqlite_schema WHERE name = ?1 ORDER BY type LIMIT 1", + [name], + ) + .await + .expect("query schema kind"); + rows.next() + .await + .expect("read schema kind") + .map(|row| row.get::(0).expect("decode schema kind")) +} + +async fn ledger_count(connection: &Connection) -> i64 { + scalar_i64(connection, "SELECT COUNT(*) FROM temper_schema_migrations").await +} + +async fn scalar_i64(connection: &Connection, sql: &str) -> i64 { + let mut rows = connection + .query(sql, ()) + .await + .expect("query integer scalar"); + rows.next() + .await + .expect("read integer scalar") + .expect("integer scalar row") + .get::(0) + .expect("integer scalar value") +} diff --git a/crates/temper-store-turso/src/migrations/schema_verify_tests/ots_probe_replacement.rs b/crates/temper-store-turso/src/migrations/schema_verify_tests/ots_probe_replacement.rs new file mode 100644 index 000000000..03e3f0cb1 --- /dev/null +++ b/crates/temper-store-turso/src/migrations/schema_verify_tests/ots_probe_replacement.rs @@ -0,0 +1,110 @@ +use libsql::params; + +use super::{ledger_count, scalar_i64, schema_kind, temporary_connection}; +use crate::migrations::catalog::{MIGRATIONS, Migration, MigrationStep}; +use crate::migrations::runner::{migrate, migrate_catalog}; +use crate::store::ots::PERSIST_OTS_TRAJECTORY_SQL; + +const OTS_PROBE_SENTINEL_STEPS: &[MigrationStep] = &[MigrationStep::Sql( + "CREATE TABLE ots_probe_migration_sentinel (id INTEGER PRIMARY KEY)", +)]; + +#[tokio::test] +async fn persisted_replacement_trigger_prevents_readiness_and_rolls_back_migration() { + let (_directory, connection) = temporary_connection("ots-persist-replacement-trigger").await; + migrate(&connection).await.expect("install current catalog"); + let trajectory_id = "existing-persisted-trajectory"; + connection + .execute( + PERSIST_OTS_TRAJECTORY_SQL, + params![ + trajectory_id, + "tenant-a", + "agent-a", + "session-before", + "outcome-before", + 1_i64, + "{\"stage\":\"before\"}", + ], + ) + .await + .expect("seed an existing trajectory through production SQL"); + connection + .execute( + "CREATE TRIGGER reject_persisted_replacement + BEFORE INSERT ON ots_trajectories + WHEN NEW.persistence_status = 'persisted' + AND EXISTS ( + SELECT 1 FROM ots_trajectories + WHERE trajectory_id = NEW.trajectory_id + ) + BEGIN SELECT RAISE(FAIL, 'persisted replacement blocked'); END", + (), + ) + .await + .expect("create replacement-only OTS trigger"); + + let runtime_error = connection + .execute( + PERSIST_OTS_TRAJECTORY_SQL, + params![ + trajectory_id, + "tenant-b", + "agent-b", + "session-after", + "outcome-after", + 2_i64, + "{\"stage\":\"after\"}", + ], + ) + .await + .expect_err("the trigger must reproduce the production replacement failure"); + assert!( + runtime_error + .to_string() + .contains("persisted replacement blocked"), + "{runtime_error}" + ); + + let mut catalog = MIGRATIONS.to_vec(); + catalog.push(Migration { + version: 8, + name: "ots-probe-transaction-sentinel", + steps: OTS_PROBE_SENTINEL_STEPS, + }); + let error = migrate_catalog(&connection, &catalog) + .await + .expect_err("readiness must reject a trigger outside the supported audit contract"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains("migration 8"), "{diagnostic}"); + assert!( + diagnostic.contains("reject_persisted_replacement"), + "{diagnostic}" + ); + assert!( + diagnostic.contains("unsupported executable trigger extension"), + "{diagnostic}" + ); + assert_eq!( + schema_kind(&connection, "ots_probe_migration_sentinel").await, + None, + "the active migration must roll back when the readiness probe fails" + ); + assert_eq!( + schema_kind(&connection, "reject_persisted_replacement") + .await + .as_deref(), + Some("trigger") + ); + assert_eq!(ledger_count(&connection).await, MIGRATIONS.len() as i64); + assert_eq!( + scalar_i64( + &connection, + "SELECT turn_count FROM ots_trajectories + WHERE trajectory_id = 'existing-persisted-trajectory'", + ) + .await, + 1, + "the rejected runtime replacement and failed probe must preserve the row" + ); +} diff --git a/crates/temper-store-turso/src/migrations/schema_verify_tests/runtime_extensions.rs b/crates/temper-store-turso/src/migrations/schema_verify_tests/runtime_extensions.rs new file mode 100644 index 000000000..488feb111 --- /dev/null +++ b/crates/temper-store-turso/src/migrations/schema_verify_tests/runtime_extensions.rs @@ -0,0 +1,581 @@ +use libsql::{Connection, params}; + +use super::{create_events, ledger_count, scalar_i64, schema_kind, temporary_connection}; +use crate::migrations::catalog::{MIGRATIONS, Migration, MigrationStep}; +use crate::migrations::runner::{migrate, migrate_catalog}; +use crate::store::ots::PERSIST_OTS_TRAJECTORY_SQL; + +const TIGHTEN_EVENTS_STEPS: &[MigrationStep] = &[MigrationStep::Sql( + "ALTER TABLE events ADD COLUMN required_value TEXT NOT NULL DEFAULT 'x'", +)]; +const DECLARED_TRIGGER_STEPS: &[MigrationStep] = &[MigrationStep::Sql( + "CREATE TRIGGER catalog_events_audit AFTER INSERT ON events BEGIN SELECT 1; END", +)]; + +#[tokio::test] +async fn later_migration_can_tighten_an_earlier_owned_table() { + let (_directory, connection) = temporary_connection("later-table-tightening").await; + migrate(&connection) + .await + .expect("install released catalog"); + assert_eq!(ledger_count(&connection).await, MIGRATIONS.len() as i64); + + let mut catalog = MIGRATIONS.to_vec(); + catalog.push(Migration { + version: 8, + name: "tighten-event-journal", + steps: TIGHTEN_EVENTS_STEPS, + }); + + migrate_catalog(&connection, &catalog) + .await + .expect("the latest catalog schema must define readiness"); + migrate_catalog(&connection, &catalog) + .await + .expect("the tightened head schema must remain replay-safe"); + + assert_eq!(ledger_count(&connection).await, 8); + assert_eq!( + scalar_i64( + &connection, + "SELECT COUNT(*) FROM pragma_table_xinfo('events') + WHERE name = 'required_value' AND \"notnull\" = 1 AND dflt_value = '''x'''", + ) + .await, + 1 + ); +} + +#[tokio::test] +async fn unexpected_trigger_prevents_ledgering_and_is_preserved() { + let (_directory, connection) = temporary_connection("unexpected-trigger").await; + create_events(&connection, None, None).await; + connection + .execute( + "CREATE TRIGGER reject_events BEFORE INSERT ON events + BEGIN SELECT RAISE(FAIL, 'blocked'); END", + (), + ) + .await + .expect("create blocking trigger"); + let before = trigger_sql(&connection, "reject_events").await; + + let error = migrate(&connection) + .await + .expect_err("an unexpected runtime-table trigger must prevent readiness"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains("migration 1"), "{diagnostic}"); + assert!(diagnostic.contains("trigger"), "{diagnostic}"); + assert_eq!(trigger_sql(&connection, "reject_events").await, before); + assert_eq!(ledger_count(&connection).await, 0); +} + +#[tokio::test] +async fn differently_cased_trigger_owner_cannot_bypass_inventory() { + let (_directory, connection) = temporary_connection("case-folded-trigger-owner").await; + create_events(&connection, None, None).await; + connection + .execute( + "CREATE TRIGGER reject_case_folded_events BEFORE INSERT ON EVENTS + BEGIN SELECT RAISE(FAIL, 'blocked'); END", + (), + ) + .await + .expect("create blocking trigger with differently cased owner"); + let before = trigger_sql(&connection, "reject_case_folded_events").await; + + let error = migrate(&connection) + .await + .expect_err("SQLite-equivalent trigger owners must not bypass readiness"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains("migration 1"), "{diagnostic}"); + assert!( + diagnostic.contains("reject_case_folded_events"), + "{diagnostic}" + ); + assert_eq!( + trigger_sql(&connection, "reject_case_folded_events").await, + before + ); + assert_eq!(ledger_count(&connection).await, 0); +} + +#[tokio::test] +async fn sqlite_x_named_trigger_cannot_bypass_inventory() { + let (_directory, connection) = temporary_connection("sqlite-x-trigger").await; + create_events(&connection, None, None).await; + connection + .execute( + "CREATE TRIGGER sqliteXreject_events BEFORE INSERT ON events + BEGIN SELECT RAISE(FAIL, 'blocked'); END", + (), + ) + .await + .expect("create legally named blocking trigger"); + + let error = migrate(&connection) + .await + .expect_err("a sqliteX-prefixed trigger must still prevent readiness"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains("migration 1"), "{diagnostic}"); + assert!(diagnostic.contains("sqliteXreject_events"), "{diagnostic}"); + assert_eq!( + schema_kind(&connection, "sqliteXreject_events") + .await + .as_deref(), + Some("trigger") + ); + assert_eq!(ledger_count(&connection).await, 0); +} + +#[tokio::test] +async fn probe_identity_bypass_trigger_prevents_readiness() { + let (_directory, connection) = temporary_connection("probe-identity-bypass-trigger").await; + migrate(&connection).await.expect("install current catalog"); + connection + .execute( + "CREATE TRIGGER reject_non_probe_tenant BEFORE INSERT ON ots_trajectories + WHEN NEW.tenant <> '__temper_trigger_probe__' + BEGIN SELECT RAISE(FAIL, 'real tenant blocked'); END", + (), + ) + .await + .expect("create trigger that recognizes the probe tenant"); + + let runtime_error = connection + .execute( + PERSIST_OTS_TRAJECTORY_SQL, + params![ + "real-trajectory", + "tenant-a", + "agent-a", + "session-a", + "outcome-a", + 1_i64, + "{}", + ], + ) + .await + .expect_err("the trigger must reproduce the real-write failure"); + assert!( + runtime_error.to_string().contains("real tenant blocked"), + "{runtime_error}" + ); + + let error = migrate(&connection) + .await + .expect_err("a trigger that distinguishes probe inputs must prevent readiness"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains("migration 7"), "{diagnostic}"); + assert!( + diagnostic.contains("reject_non_probe_tenant"), + "{diagnostic}" + ); + assert_eq!(ledger_count(&connection).await, MIGRATIONS.len() as i64); +} + +#[tokio::test] +async fn unmodeled_ots_trigger_mutation_prevents_readiness() { + let (_directory, connection) = temporary_connection("unmodeled-ots-trigger-mutation").await; + migrate(&connection).await.expect("install current catalog"); + connection + .execute( + "CREATE TRIGGER mutate_ots_entity_type AFTER INSERT ON ots_trajectories + BEGIN + UPDATE ots_trajectories + SET entity_type = 'trigger-corruption' + WHERE trajectory_id = NEW.trajectory_id; + END", + (), + ) + .await + .expect("create trigger that mutates an unasserted OTS column"); + + connection + .execute( + PERSIST_OTS_TRAJECTORY_SQL, + params![ + "mutated-trajectory", + "tenant-a", + "agent-a", + "session-a", + "outcome-a", + 1_i64, + "{}", + ], + ) + .await + .expect("the trigger leaves the currently asserted production fields writable"); + assert_eq!( + scalar_text( + &connection, + "SELECT entity_type FROM ots_trajectories + WHERE trajectory_id = 'mutated-trajectory'", + ) + .await + .as_deref(), + Some("trigger-corruption"), + "the trigger must reproduce an unasserted mutation" + ); + + let error = migrate(&connection) + .await + .expect_err("a trigger outside the supported audit contract must prevent readiness"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains("migration 7"), "{diagnostic}"); + assert!( + diagnostic.contains("mutate_ots_entity_type"), + "{diagnostic}" + ); + assert_eq!(ledger_count(&connection).await, MIGRATIONS.len() as i64); +} + +#[tokio::test] +async fn blocking_legacy_ots_trigger_fails_the_runtime_write_probe() { + let (_directory, connection) = temporary_connection("blocking-ots-trigger").await; + migrate(&connection).await.expect("install current catalog"); + connection + .execute( + "CREATE TRIGGER reject_ots_insert AFTER INSERT ON ots_trajectories + BEGIN SELECT RAISE(FAIL, 'blocked'); END", + (), + ) + .await + .expect("create blocking OTS trigger"); + let before = trigger_sql(&connection, "reject_ots_insert").await; + + let error = migrate(&connection) + .await + .expect_err("a trigger that rejects canonical OTS writes must prevent readiness"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains("migration 7"), "{diagnostic}"); + assert!(diagnostic.contains("reject_ots_insert"), "{diagnostic}"); + assert!( + diagnostic.contains("unsupported executable trigger extension"), + "{diagnostic}" + ); + assert_eq!(trigger_sql(&connection, "reject_ots_insert").await, before); + assert_eq!(ledger_count(&connection).await, MIGRATIONS.len() as i64); +} + +#[tokio::test] +async fn queued_only_ots_trigger_fails_the_production_write_probe() { + let (_directory, connection) = temporary_connection("queued-ots-trigger").await; + migrate(&connection).await.expect("install current catalog"); + connection + .execute( + "CREATE TRIGGER reject_queued_ots BEFORE INSERT ON ots_trajectories + WHEN NEW.persistence_status = 'queued' + BEGIN SELECT RAISE(FAIL, 'queued blocked'); END", + (), + ) + .await + .expect("create queued-only OTS trigger"); + + let error = migrate(&connection) + .await + .expect_err("the probe must exercise the production queued insert path"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains("migration 7"), "{diagnostic}"); + assert!(diagnostic.contains("reject_queued_ots"), "{diagnostic}"); + assert!( + diagnostic.contains("unsupported executable trigger extension"), + "{diagnostic}" + ); + assert_eq!(ledger_count(&connection).await, MIGRATIONS.len() as i64); +} + +#[tokio::test] +async fn benign_legacy_ots_trigger_probe_has_no_durable_side_effects() { + let (_directory, connection) = temporary_connection("benign-ots-trigger").await; + migrate(&connection).await.expect("install current catalog"); + connection + .execute( + "CREATE TABLE ots_probe_audit (trajectory_id TEXT PRIMARY KEY)", + (), + ) + .await + .expect("create OTS audit table"); + connection + .execute( + "CREATE TRIGGER audit_ots_insert AFTER INSERT ON ots_trajectories + BEGIN + INSERT INTO ots_probe_audit (trajectory_id) VALUES (NEW.trajectory_id); + END", + (), + ) + .await + .expect("create benign OTS trigger"); + + migrate(&connection) + .await + .expect("a canonical-write-compatible OTS trigger must remain supported"); + assert_eq!( + scalar_i64(&connection, "SELECT COUNT(*) FROM ots_probe_audit").await, + 0, + "the validation probe and its trigger side effects must roll back" + ); + + connection + .execute( + "INSERT INTO ots_trajectories (trajectory_id, tenant, agent_id, data) + VALUES ('runtime-trajectory', 'tenant-a', 'agent-a', '{}')", + (), + ) + .await + .expect("runtime write through validated OTS trigger"); + assert_eq!( + scalar_i64( + &connection, + "SELECT COUNT(*) FROM ots_probe_audit + WHERE trajectory_id = 'runtime-trajectory'", + ) + .await, + 1 + ); + assert_eq!(ledger_count(&connection).await, MIGRATIONS.len() as i64); +} + +#[tokio::test] +async fn ots_audit_trigger_with_executable_sink_restriction_prevents_readiness() { + let (_directory, connection) = temporary_connection("restricted-ots-audit-sink").await; + migrate(&connection).await.expect("install current catalog"); + connection + .execute( + "CREATE TABLE ots_restricted_audit ( + trajectory_id TEXT PRIMARY KEY, + CHECK (trajectory_id GLOB '__temper_trigger_probe__-*') + )", + (), + ) + .await + .expect("create audit sink that recognizes historical probe ids"); + connection + .execute( + "CREATE TRIGGER audit_ots_restricted AFTER INSERT ON ots_trajectories + BEGIN + INSERT INTO ots_restricted_audit (trajectory_id) + VALUES (NEW.trajectory_id); + END", + (), + ) + .await + .expect("create structurally simple trigger with an executable sink"); + + let error = migrate(&connection) + .await + .expect_err("an executable audit sink schema must prevent readiness"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains("migration 7"), "{diagnostic}"); + assert!(diagnostic.contains("audit_ots_restricted"), "{diagnostic}"); + assert!( + diagnostic.contains("unsupported audit sink"), + "{diagnostic}" + ); + assert_eq!(ledger_count(&connection).await, MIGRATIONS.len() as i64); +} + +#[tokio::test] +async fn unexpected_expression_index_prevents_ledgering_and_is_preserved() { + let (_directory, connection) = temporary_connection("unexpected-expression-index").await; + create_events(&connection, None, None).await; + connection + .execute( + "CREATE INDEX events_unexpected_expression + ON events(json_extract(payload, 'invalid-path'))", + (), + ) + .await + .expect("create executable expression index"); + + let error = migrate(&connection) + .await + .expect_err("an executable expression index must prevent readiness"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains("migration 1"), "{diagnostic}"); + assert!( + diagnostic.contains("events_unexpected_expression"), + "{diagnostic}" + ); + assert_eq!( + schema_kind(&connection, "events_unexpected_expression") + .await + .as_deref(), + Some("index") + ); + assert_eq!(ledger_count(&connection).await, 0); +} + +#[tokio::test] +async fn sqlite_x_named_expression_index_cannot_bypass_inventory() { + let (_directory, connection) = temporary_connection("sqlite-x-expression-index").await; + create_events(&connection, None, None).await; + connection + .execute( + "CREATE INDEX sqliteXevents_expression + ON events(json_extract(payload, 'invalid-path'))", + (), + ) + .await + .expect("create legally named expression index"); + + let error = migrate(&connection) + .await + .expect_err("a sqliteX-prefixed expression index must prevent readiness"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains("migration 1"), "{diagnostic}"); + assert!( + diagnostic.contains("sqliteXevents_expression"), + "{diagnostic}" + ); + assert_eq!( + schema_kind(&connection, "sqliteXevents_expression") + .await + .as_deref(), + Some("index") + ); + assert_eq!(ledger_count(&connection).await, 0); +} + +#[tokio::test] +async fn unexpected_partial_index_prevents_ledgering_and_is_preserved() { + let (_directory, connection) = temporary_connection("unexpected-partial-index").await; + create_events(&connection, None, None).await; + connection + .execute( + "CREATE INDEX events_unexpected_partial ON events(event_type) + WHERE json_extract(payload, 'invalid-path')", + (), + ) + .await + .expect("create executable partial index"); + + let error = migrate(&connection) + .await + .expect_err("an executable partial index must prevent readiness"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains("migration 1"), "{diagnostic}"); + assert!( + diagnostic.contains("events_unexpected_partial"), + "{diagnostic}" + ); + assert_eq!( + schema_kind(&connection, "events_unexpected_partial") + .await + .as_deref(), + Some("index") + ); + assert_eq!(ledger_count(&connection).await, 0); +} + +#[tokio::test] +async fn plain_non_unique_index_extension_remains_compatible() { + let (_directory, connection) = temporary_connection("plain-index-extension").await; + create_events(&connection, None, None).await; + connection + .execute( + "CREATE INDEX events_deployment_lookup ON events(event_type DESC, tenant)", + (), + ) + .await + .expect("create plain index extension"); + + migrate(&connection) + .await + .expect("a plain non-unique index extension must remain compatible"); + connection + .execute( + "INSERT INTO events ( + tenant, entity_type, entity_id, sequence_nr, event_type, payload + ) VALUES ('tenant-a', 'Order', 'order-1', 1, 'Created', '{}')", + (), + ) + .await + .expect("canonical runtime insert with plain index extension"); + assert_eq!(ledger_count(&connection).await, MIGRATIONS.len() as i64); +} + +#[tokio::test] +async fn declared_trigger_mismatch_and_absence_prevent_replay() { + let (_directory, connection) = temporary_connection("declared-trigger").await; + migrate(&connection) + .await + .expect("install released catalog"); + let mut catalog = MIGRATIONS.to_vec(); + catalog.push(Migration { + version: 8, + name: "declare-events-trigger", + steps: DECLARED_TRIGGER_STEPS, + }); + migrate_catalog(&connection, &catalog) + .await + .expect("install declared trigger migration"); + + connection + .execute("DROP TRIGGER catalog_events_audit", ()) + .await + .expect("drop declared trigger"); + connection + .execute( + "CREATE TRIGGER catalog_events_audit AFTER DELETE ON events BEGIN SELECT 1; END", + (), + ) + .await + .expect("install mismatched trigger definition"); + let mismatch = migrate_catalog(&connection, &catalog) + .await + .expect_err("a changed declared trigger must prevent readiness"); + let mismatch_diagnostic = mismatch.to_string(); + assert!( + mismatch_diagnostic.contains("migration 8"), + "{mismatch_diagnostic}" + ); + assert!( + mismatch_diagnostic.contains("incompatible semantics"), + "{mismatch_diagnostic}" + ); + + connection + .execute("DROP TRIGGER catalog_events_audit", ()) + .await + .expect("remove mismatched trigger"); + let missing = migrate_catalog(&connection, &catalog) + .await + .expect_err("a missing declared trigger must prevent readiness"); + let missing_diagnostic = missing.to_string(); + assert!( + missing_diagnostic.contains("migration 8"), + "{missing_diagnostic}" + ); + assert!( + missing_diagnostic.contains("missing required trigger"), + "{missing_diagnostic}" + ); + assert_eq!(ledger_count(&connection).await, 8); +} + +async fn trigger_sql(connection: &Connection, trigger: &str) -> String { + let mut rows = connection + .query( + "SELECT sql FROM sqlite_schema WHERE type = 'trigger' AND name = ?1", + [trigger], + ) + .await + .expect("query trigger SQL"); + rows.next() + .await + .expect("read trigger SQL") + .expect("trigger SQL row") + .get::(0) + .expect("decode trigger SQL") +} + +async fn scalar_text(connection: &Connection, sql: &str) -> Option { + let mut rows = connection.query(sql, ()).await.expect("query text scalar"); + rows.next() + .await + .expect("read text scalar") + .expect("text scalar row") + .get::>(0) + .expect("decode text scalar") +} diff --git a/crates/temper-store-turso/src/migrations/tests.rs b/crates/temper-store-turso/src/migrations/tests.rs new file mode 100644 index 000000000..275a897b6 --- /dev/null +++ b/crates/temper-store-turso/src/migrations/tests.rs @@ -0,0 +1,500 @@ +use std::process::{Command, Stdio}; + +use libsql::{Builder, Connection, params}; + +use super::catalog::MIGRATIONS; +use super::runner::{FaultInjection, expected_checksums, migrate, migrate_prefix}; +use crate::TursoEventStore; + +#[tokio::test] +async fn migration_catalog_is_contiguous_and_checksummed() { + let checksums = expected_checksums().await.expect("expected checksums"); + for (index, migration) in MIGRATIONS.iter().enumerate() { + assert_eq!(migration.version, (index + 1) as u32); + assert_eq!(checksums[index].len(), 64); + } +} + +#[tokio::test] +async fn fresh_install_records_every_migration_and_duplicate_replay_is_stable() { + let (_directory, connection) = temporary_connection("fresh").await; + migrate(&connection).await.expect("fresh migration"); + let before = ledger_rows(&connection).await; + let checksums = expected_checksums().await.expect("expected checksums"); + assert_eq!(before.len(), MIGRATIONS.len()); + for ((row, migration), checksum) in before.iter().zip(MIGRATIONS).zip(checksums) { + assert_eq!(row.0, migration.version as i64); + assert_eq!(row.1, migration.name); + assert_eq!(row.2, checksum); + assert_eq!(row.2.len(), 64); + } + migrate(&connection).await.expect("idempotent replay"); + assert_eq!(ledger_rows(&connection).await, before); +} + +#[tokio::test] +async fn every_supported_version_prefix_upgrades_to_latest() { + for prefix in 0..=MIGRATIONS.len() { + let (_directory, connection) = temporary_connection(&format!("prefix-{prefix}")).await; + migrate_prefix(&connection, prefix, FaultInjection::default()) + .await + .unwrap_or_else(|error| panic!("install prefix {prefix}: {error}")); + migrate(&connection) + .await + .unwrap_or_else(|error| panic!("upgrade prefix {prefix}: {error}")); + assert_eq!(ledger_rows(&connection).await.len(), MIGRATIONS.len()); + } +} + +#[tokio::test] +async fn legacy_partial_schema_is_reconciled_without_losing_rows() { + let (_directory, connection) = temporary_connection("legacy-partial").await; + connection + .execute( + "CREATE TABLE events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + tenant TEXT NOT NULL, + entity_type TEXT NOT NULL, + entity_id TEXT NOT NULL, + sequence_nr INTEGER NOT NULL, + event_type TEXT NOT NULL, + payload TEXT NOT NULL, + metadata TEXT, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + UNIQUE(tenant, entity_type, entity_id, sequence_nr) + )", + (), + ) + .await + .expect("legacy events table"); + install_legacy_ots_table(&connection).await; + connection + .execute( + "CREATE TABLE ots_insert_audit (trajectory_id TEXT PRIMARY KEY)", + (), + ) + .await + .expect("legacy OTS audit table"); + connection + .execute( + "CREATE INDEX idx_ots_legacy_session ON ots_trajectories(session_id)", + (), + ) + .await + .expect("legacy OTS custom index"); + connection + .execute( + "CREATE TRIGGER trg_ots_legacy_insert AFTER INSERT ON ots_trajectories + BEGIN INSERT INTO ots_insert_audit (trajectory_id) VALUES (NEW.trajectory_id); END", + (), + ) + .await + .expect("legacy OTS custom trigger"); + connection + .execute( + "INSERT INTO ots_trajectories ( + trajectory_id, tenant, agent_id, data, created_at + ) VALUES ('trajectory-1', 'tenant-a', 'agent-a', '{\"turns\":1}', '2026-07-14T00:00:00Z')", + (), + ) + .await + .expect("legacy OTS row"); + migrate(&connection).await.expect("legacy reconciliation"); + assert!( + column_names(&connection, "events") + .await + .iter() + .any(|column| column == "segment_index") + ); + let mut rows = connection + .query( + "SELECT data, persistence_status, persist_attempts, last_error, updated_at + FROM ots_trajectories WHERE trajectory_id = 'trajectory-1'", + (), + ) + .await + .expect("query reconciled OTS row"); + let row = rows + .next() + .await + .expect("read reconciled OTS row") + .expect("reconciled OTS row exists"); + assert_eq!(row.get::(0).expect("data"), "{\"turns\":1}"); + assert_eq!( + row.get::(1).expect("persistence status"), + "persisted" + ); + assert_eq!(row.get::(2).expect("persist attempts"), 0); + assert_eq!(row.get::>(3).expect("last error"), None); + assert_eq!( + row.get::(4).expect("updated at"), + "2026-07-14T00:00:00Z" + ); + assert_eq!( + schema_kind(&connection, "idx_ots_legacy_session") + .await + .as_deref(), + Some("index") + ); + assert_eq!( + schema_kind(&connection, "trg_ots_legacy_insert") + .await + .as_deref(), + Some("trigger") + ); + connection + .execute( + "INSERT INTO ots_trajectories (trajectory_id, tenant, agent_id, data) + VALUES ('trajectory-2', 'tenant-a', 'agent-a', '{}')", + (), + ) + .await + .expect("insert through preserved OTS trigger"); + assert_eq!( + scalar_count( + &connection, + "SELECT COUNT(*) FROM ots_insert_audit WHERE trajectory_id = 'trajectory-2'" + ) + .await, + 1 + ); + assert_eq!(ledger_rows(&connection).await.len(), MIGRATIONS.len()); +} + +#[tokio::test] +async fn legacy_ots_extra_column_fails_closed_without_mutation() { + let (_directory, connection) = temporary_connection("legacy-ots-extra").await; + install_legacy_ots_table(&connection).await; + connection + .execute( + "ALTER TABLE ots_trajectories ADD COLUMN deployment_note TEXT", + (), + ) + .await + .expect("legacy custom OTS column"); + connection + .execute( + "INSERT INTO ots_trajectories (trajectory_id, tenant, agent_id, data, deployment_note) + VALUES ('trajectory-extra', 'tenant-a', 'agent-a', '{}', 'preserve-me')", + (), + ) + .await + .expect("legacy custom OTS row"); + let error = migrate(&connection) + .await + .expect_err("an unmodeled legacy column must prevent destructive reconciliation"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains("migration 5"), "{diagnostic}"); + assert!( + diagnostic.contains("exactly the pre-upgrade columns"), + "{diagnostic}" + ); + assert!( + column_names(&connection, "ots_trajectories") + .await + .iter() + .any(|column| column == "deployment_note") + ); + assert_eq!( + scalar_count( + &connection, + "SELECT COUNT(*) FROM ots_trajectories + WHERE trajectory_id = 'trajectory-extra' AND deployment_note = 'preserve-me'" + ) + .await, + 1 + ); + assert_eq!(ledger_rows(&connection).await.len(), 4); +} + +#[tokio::test] +async fn interrupted_migration_rolls_back_schema_and_ledger_then_retries() { + let (_directory, connection) = temporary_connection("interrupted").await; + let error = migrate_prefix( + &connection, + 1, + FaultInjection { + after_step: Some((1, 0)), + ..FaultInjection::default() + }, + ) + .await + .expect_err("fault injection must interrupt migration"); + assert!( + error + .to_string() + .contains("injected migration interruption") + ); + assert_eq!(schema_kind(&connection, "events").await, None); + assert!(ledger_rows(&connection).await.is_empty()); + + migrate(&connection).await.expect("retry after rollback"); + assert_eq!( + schema_kind(&connection, "events").await.as_deref(), + Some("table") + ); + assert_eq!(ledger_rows(&connection).await.len(), MIGRATIONS.len()); +} + +#[tokio::test] +async fn backend_ddl_error_rolls_back_active_version_with_context() { + let (_directory, connection) = temporary_connection("ddl-error").await; + let error = migrate_prefix( + &connection, + 1, + FaultInjection { + ddl_error_at: Some((1, 1)), + ..FaultInjection::default() + }, + ) + .await + .expect_err("an actual libSQL DDL error must prevent readiness"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains("migration 1"), "{diagnostic}"); + assert!( + diagnostic.contains("event-journal-and-snapshots"), + "{diagnostic}" + ); + assert!(diagnostic.contains("step 1"), "{diagnostic}"); + assert!( + diagnostic.contains("__temper_injected_missing_table"), + "{diagnostic}" + ); + assert_eq!(schema_kind(&connection, "events").await, None); + assert!(ledger_rows(&connection).await.is_empty()); + + migrate(&connection) + .await + .expect("retry after backend DDL failure"); + assert_eq!(ledger_rows(&connection).await.len(), MIGRATIONS.len()); +} + +#[tokio::test] +async fn checksum_gap_and_incompatible_newer_versions_fail_closed() { + let (_directory, connection) = temporary_connection("ledger-corruption").await; + migrate(&connection).await.expect("initial migration"); + let checksums = expected_checksums().await.expect("expected checksums"); + + connection + .execute( + "UPDATE temper_schema_migrations SET checksum = ?1 WHERE version = 1", + ["0".repeat(64)], + ) + .await + .expect("alter checksum"); + let error = migrate(&connection) + .await + .expect_err("checksum drift must fail startup"); + assert!(error.to_string().contains("checksum mismatch")); + connection + .execute( + "UPDATE temper_schema_migrations SET checksum = ?1 WHERE version = 1", + [checksums[0].as_str()], + ) + .await + .expect("restore checksum"); + + connection + .execute("DELETE FROM temper_schema_migrations WHERE version = 3", ()) + .await + .expect("create ledger gap"); + let error = migrate(&connection) + .await + .expect_err("ledger gap must fail startup"); + assert!(error.to_string().contains("version gap")); + connection + .execute( + "INSERT INTO temper_schema_migrations (version, name, checksum) + VALUES (?1, ?2, ?3)", + params![ + MIGRATIONS[2].version as i64, + MIGRATIONS[2].name, + checksums[2].as_str() + ], + ) + .await + .expect("restore missing ledger row"); + + connection + .execute( + "INSERT INTO temper_schema_migrations (version, name, checksum) + VALUES (8, 'future-schema', ?1)", + ["f".repeat(64)], + ) + .await + .expect("install newer ledger row"); + let error = migrate(&connection) + .await + .expect_err("newer schema must fail startup"); + assert!( + error + .to_string() + .contains("newer than this binary supports") + ); +} + +#[tokio::test] +async fn semantic_index_drift_prevents_readiness() { + let (_directory, connection) = temporary_connection("semantic-drift").await; + migrate(&connection).await.expect("initial migration"); + connection + .execute("DROP INDEX idx_events_entity", ()) + .await + .expect("drop expected index"); + connection + .execute("CREATE INDEX idx_events_entity ON events(event_type)", ()) + .await + .expect("install incompatible index"); + let error = migrate(&connection) + .await + .expect_err("semantic index drift must fail readiness"); + let diagnostic = error.to_string(); + assert!(diagnostic.contains("idx_events_entity"), "{diagnostic}"); + assert!( + diagnostic.contains("incompatible semantics"), + "{diagnostic}" + ); + assert!(diagnostic.contains("migration 7"), "{diagnostic}"); +} + +#[test] +fn independent_startup_processes_converge_on_one_ledger() { + let directory = tempfile::tempdir().expect("temporary database directory"); + let executable = std::env::current_exe().expect("current test executable"); + let runtime = tokio::runtime::Runtime::new().expect("test runtime"); + for round in 0..8 { + let database_path = directory.path().join(format!("multiprocess-{round}.db")); + let url = format!("file:{}", database_path.display()); + let spawn_child = || { + Command::new(&executable) + .args([ + "--ignored", + "--exact", + "migrations::tests::migration_process_child", + "--nocapture", + ]) + .env("TEMPER_MIGRATION_TEST_URL", &url) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .expect("spawn migration child") + }; + let first = spawn_child(); + let second = spawn_child(); + let first_output = first.wait_with_output().expect("wait for first child"); + let second_output = second.wait_with_output().expect("wait for second child"); + assert_child_succeeded("first", &first_output); + assert_child_succeeded("second", &second_output); + runtime.block_on(async { + let database = Builder::new_local(&database_path) + .build() + .await + .expect("open migrated database"); + let connection = database.connect().expect("connect to migrated database"); + assert_eq!(ledger_rows(&connection).await.len(), MIGRATIONS.len()); + }); + } +} + +#[tokio::test] +#[ignore = "helper process invoked by independent_startup_processes_converge_on_one_ledger"] +async fn migration_process_child() { + let url = std::env::var("TEMPER_MIGRATION_TEST_URL").expect("migration child URL"); + TursoEventStore::new(&url, None) + .await + .expect("independent store startup"); +} + +async fn install_legacy_ots_table(connection: &Connection) { + connection + .execute( + "CREATE TABLE ots_trajectories ( + trajectory_id TEXT PRIMARY KEY, + tenant TEXT NOT NULL, + agent_id TEXT NOT NULL, + session_id TEXT, + outcome TEXT NOT NULL DEFAULT 'unknown', + entity_type TEXT, + turn_count INTEGER NOT NULL DEFAULT 0, + data TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (datetime('now')) + )", + (), + ) + .await + .expect("legacy OTS table"); +} + +async fn temporary_connection(label: &str) -> (tempfile::TempDir, Connection) { + let directory = tempfile::tempdir().expect("temporary database directory"); + let database_path = directory.path().join(format!("{label}.db")); + let database = Builder::new_local(database_path) + .build() + .await + .expect("create temporary database"); + let connection = database.connect().expect("connect to temporary database"); + (directory, connection) +} + +async fn ledger_rows(connection: &Connection) -> Vec<(i64, String, String, String)> { + let mut rows = connection + .query( + "SELECT version, name, checksum, applied_at + FROM temper_schema_migrations ORDER BY version", + (), + ) + .await + .expect("query migration ledger"); + let mut values = Vec::new(); + while let Some(row) = rows.next().await.expect("read migration ledger") { + values.push(( + row.get::(0).expect("version"), + row.get::(1).expect("name"), + row.get::(2).expect("checksum"), + row.get::(3).expect("applied at"), + )); + } + values +} + +async fn scalar_count(connection: &Connection, sql: &str) -> i64 { + let mut rows = connection.query(sql, ()).await.expect("query scalar count"); + rows.next() + .await + .expect("read scalar count") + .expect("scalar count row") + .get::(0) + .expect("scalar count value") +} + +async fn column_names(connection: &Connection, table: &str) -> Vec { + let mut rows = connection + .query(&format!("PRAGMA table_info(\"{table}\")"), ()) + .await + .expect("query table columns"); + let mut names = Vec::new(); + while let Some(row) = rows.next().await.expect("read table column") { + names.push(row.get::(1).expect("column name")); + } + names +} + +async fn schema_kind(connection: &Connection, name: &str) -> Option { + let mut rows = connection + .query("SELECT type FROM sqlite_schema WHERE name = ?1", [name]) + .await + .expect("query schema kind"); + rows.next() + .await + .expect("read schema kind") + .map(|row| row.get::(0).expect("schema kind")) +} + +fn assert_child_succeeded(label: &str, output: &std::process::Output) { + assert!( + output.status.success(), + "{label} migration child failed\nstdout:\n{}\nstderr:\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); +} diff --git a/crates/temper-store-turso/src/router.rs b/crates/temper-store-turso/src/router.rs index 3c7e1fe01..adcd18f4d 100644 --- a/crates/temper-store-turso/src/router.rs +++ b/crates/temper-store-turso/src/router.rs @@ -21,7 +21,6 @@ use temper_runtime::persistence::{ use temper_runtime::tenant::parse_persistence_id_parts; use crate::TursoEventStore; -use crate::schema; /// Routes storage operations to per-tenant Turso databases. /// @@ -91,9 +90,6 @@ impl TenantStoreRouter { ) -> Result { let platform = TursoEventStore::new(platform_url, platform_token).await?; - // Run platform-specific migrations (tenant registry + user tables). - Self::migrate_platform(&platform).await?; - let router = Self { platform, tenants: Arc::new(RwLock::new(BTreeMap::new())), @@ -380,21 +376,6 @@ impl TenantStoreRouter { // ── Private helpers ────────────────────────────────────────────────── - /// Run platform-specific schema migrations. - async fn migrate_platform(store: &TursoEventStore) -> Result<(), PersistenceError> { - let conn = store.connection().map_err(storage_error)?; - conn.execute(schema::CREATE_TENANT_REGISTRY_TABLE, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_TENANT_USERS_TABLE, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_TENANT_USERS_USER_INDEX, ()) - .await - .map_err(storage_error)?; - Ok(()) - } - /// Load all tenant registry rows from the platform DB. async fn load_tenant_registry(&self) -> Result, PersistenceError> { let conn = self.platform.connection().map_err(storage_error)?; diff --git a/crates/temper-store-turso/src/store/mod.rs b/crates/temper-store-turso/src/store/mod.rs index fba0d839d..d1115a9fc 100644 --- a/crates/temper-store-turso/src/store/mod.rs +++ b/crates/temper-store-turso/src/store/mod.rs @@ -1,13 +1,4 @@ //! Turso/libSQL-backed implementation of the [`EventStore`] trait. -//! -//! Split into domain-focused sub-modules for cohesion: -//! - [`specs`]: Spec CRUD (upsert, verification, load) -//! - [`trajectory`]: Trajectory persistence and queries -//! - [`evolution`]: Feature requests, evolution records, design-time events -//! - [`authz`]: Authorization decisions and Cedar policies -//! - [`wasm`]: WASM module storage and invocation logs -//! - [`constraints`]: Tenant-level cross-entity constraints -//! - [`event_store`]: [`EventStore`] trait implementation use libsql::{Builder, Database}; use std::sync::Arc; @@ -16,8 +7,6 @@ use temper_runtime::persistence::{PersistenceError, storage_error}; use tokio::sync::Semaphore; use tracing::instrument; -use crate::schema; - mod append_config; mod authz; mod blobs; @@ -107,7 +96,7 @@ impl TursoEventStore { Ok(conn) } - /// Run schema migrations on connect. + /// Run ordered, checksummed schema migrations on connect. #[instrument(skip_all, fields(otel.name = "turso.migrate"))] async fn migrate(&self) -> Result<(), PersistenceError> { let conn = self.connection()?; @@ -115,300 +104,19 @@ impl TursoEventStore { // PRAGMAs are SQLite-specific and not supported on remote Turso Cloud. // Turso Cloud manages its own journal mode and concurrency. if !self.is_remote { + // Install the busy handler before WAL setup for simultaneous opens. let _ = conn - .query("PRAGMA journal_mode=WAL", ()) + .query("PRAGMA busy_timeout=30000", ()) .await .map_err(storage_error)?; let _ = conn - .query("PRAGMA busy_timeout=30000", ()) + .query("PRAGMA journal_mode=WAL", ()) .await .map_err(storage_error)?; } - conn.execute(schema::CREATE_EVENTS_TABLE, ()) - .await - .map_err(storage_error)?; - let _ = conn - .execute(schema::ALTER_EVENTS_ADD_SEGMENT_INDEX, ()) - .await; - conn.execute(schema::CREATE_EVENTS_ENTITY_INDEX, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_EVENT_SEGMENTS_TABLE, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_EVENT_SEGMENTS_OPEN_INDEX, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_SNAPSHOTS_TABLE, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_SNAPSHOT_HISTORY_TABLE, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_SNAPSHOT_HISTORY_ENTITY_INDEX, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_SPECS_TABLE, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_TRAJECTORIES_TABLE, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_TRAJECTORIES_SUCCESS_INDEX, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_TRAJECTORIES_ENTITY_ACTION_INDEX, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_TENANT_CONSTRAINTS_TABLE, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_WASM_MODULES_TABLE, ()) - .await - .map_err(storage_error)?; - // Idempotent ALTER for pre-existing DBs created before the source - // column existed. Turso has no IF NOT EXISTS for ADD COLUMN, so we - // ignore "duplicate column" errors. - match conn - .execute(schema::ADD_WASM_MODULES_SOURCE_COLUMN, ()) - .await - { - Ok(_) => {} - Err(e) => { - let msg = e.to_string(); - if !msg.contains("duplicate column") - && !msg.contains("already exists") - && !msg.contains("already has") - { - return Err(storage_error(e)); - } - } - } - conn.execute(schema::CREATE_WASM_INVOCATION_LOGS_TABLE, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_WASM_INVOCATION_LOGS_TENANT_INDEX, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_WASM_INVOCATION_LOGS_MODULE_INDEX, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_WASM_INVOCATION_LOGS_CREATED_INDEX, ()) - .await - .map_err(storage_error)?; - - conn.execute(schema::CREATE_PENDING_DECISIONS_TABLE, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_PENDING_DECISIONS_TENANT_INDEX, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_PENDING_DECISIONS_STATUS_INDEX, ()) - .await - .map_err(storage_error)?; - - conn.execute(schema::CREATE_TENANT_POLICIES_TABLE, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_POLICIES_TABLE, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_POLICY_DENIAL_PATTERNS_TABLE, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_POLICY_DENIAL_PATTERNS_TENANT_INDEX, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_PUBLISHED_ARTIFACTS_TABLE, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_PUBLISHED_ARTIFACTS_OWNER_INDEX, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_PUBLISHED_ARTIFACTS_SOURCE_INDEX, ()) - .await - .map_err(storage_error)?; - // Migration: add `enabled` column to existing `policies` tables. - let _ = conn.execute(schema::ALTER_POLICIES_ADD_ENABLED, ()).await; - conn.execute(schema::CREATE_TENANT_INSTALLED_APPS_TABLE, ()) - .await - .map_err(storage_error)?; - for stmt in [ - schema::ALTER_INSTALLED_APPS_ADD_APP_VERSION, - schema::ALTER_INSTALLED_APPS_ADD_SOURCE_KIND, - schema::ALTER_INSTALLED_APPS_ADD_APP_REF, - schema::ALTER_INSTALLED_APPS_ADD_VERSION_HASH, - schema::ALTER_INSTALLED_APPS_ADD_PINNED_VERSION_HASH, - schema::ALTER_INSTALLED_APPS_ADD_CURRENT_VERSION_HASH, - schema::ALTER_INSTALLED_APPS_ADD_FOLLOW_POLICY, - schema::ALTER_INSTALLED_APPS_ADD_CLOSURE_ID, - schema::ALTER_INSTALLED_APPS_ADD_REGISTRY_URL, - schema::ALTER_INSTALLED_APPS_ADD_REGISTRY_TENANT, - schema::ALTER_INSTALLED_APPS_ADD_BUNDLE_DIGEST, - schema::ALTER_INSTALLED_APPS_ADD_SPEC_DIGEST, - schema::ALTER_INSTALLED_APPS_ADD_POLICY_DIGEST, - schema::ALTER_INSTALLED_APPS_ADD_WASM_DIGEST, - schema::ALTER_INSTALLED_APPS_ADD_CONTENT_DIGEST, - schema::ALTER_INSTALLED_APPS_ADD_SEED_DIGEST, - schema::ALTER_INSTALLED_APPS_ADD_LAST_RECONCILED_AT, - schema::ALTER_INSTALLED_APPS_ADD_STATUS, - ] { - let _ = conn.execute(stmt, ()).await; - } - - // Phase 0: New tables for Turso-as-single-source-of-truth. - conn.execute(schema::CREATE_FEATURE_REQUESTS_TABLE, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_EVOLUTION_RECORDS_TABLE, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_EVOLUTION_RECORDS_TYPE_INDEX, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_EVOLUTION_RECORDS_STATUS_INDEX, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_DESIGN_TIME_EVENTS_TABLE, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_DESIGN_TIME_EVENTS_TENANT_INDEX, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_TENANT_SECRETS_TABLE, ()) - .await - .map_err(storage_error)?; - - conn.execute(schema::CREATE_TENANT_SECRETS_TABLE, ()) - .await - .map_err(storage_error)?; - - // Specs table extensions — add content_hash column for verification caching. - let _ = conn.execute(schema::ALTER_SPECS_ADD_CONTENT_HASH, ()).await; - let _ = conn.execute(schema::ALTER_SPECS_ADD_COMMITTED, ()).await; - - // Trajectory table extensions — ALTER TABLE to add missing columns. - // SQLite returns an error for duplicate columns, so we ignore failures. - for stmt in &[ - schema::ALTER_TRAJECTORIES_ADD_AGENT_ID, - schema::ALTER_TRAJECTORIES_ADD_SESSION_ID, - schema::ALTER_TRAJECTORIES_ADD_AUTHZ_DENIED, - schema::ALTER_TRAJECTORIES_ADD_DENIED_RESOURCE, - schema::ALTER_TRAJECTORIES_ADD_DENIED_MODULE, - schema::ALTER_TRAJECTORIES_ADD_SOURCE, - schema::ALTER_TRAJECTORIES_ADD_SPEC_GOVERNED, - schema::ALTER_TRAJECTORIES_ADD_REQUEST_BODY, - schema::ALTER_TRAJECTORIES_ADD_INTENT, - schema::ALTER_TRAJECTORIES_ADD_MATCHED_POLICY_IDS, - ] { - let _ = conn.execute(stmt, ()).await; // ignore "duplicate column" errors - } - conn.execute(schema::CREATE_TRAJECTORIES_AGENT_INDEX, ()) - .await - .map_err(storage_error)?; - - // OTS trajectory storage — full agent execution traces for GEPA. - conn.execute(schema::CREATE_OTS_TRAJECTORIES_TABLE, ()) - .await - .map_err(storage_error)?; - for stmt in [ - schema::ALTER_OTS_TRAJECTORIES_ADD_PERSISTENCE_STATUS, - schema::ALTER_OTS_TRAJECTORIES_ADD_PERSIST_ATTEMPTS, - schema::ALTER_OTS_TRAJECTORIES_ADD_LAST_ERROR, - schema::ALTER_OTS_TRAJECTORIES_ADD_UPDATED_AT, - ] { - let _ = conn.execute(stmt, ()).await; - } - conn.execute(schema::CREATE_OTS_TRAJECTORIES_AGENT_INDEX, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_OTS_TRAJECTORIES_TENANT_INDEX, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_OTS_TRAJECTORIES_OUTCOME_INDEX, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_OTS_TRAJECTORIES_STATUS_INDEX, ()) - .await - .map_err(storage_error)?; - - // Blob storage — content-addressed binary objects for TemperFS and - // field-overflow blob refs (ADR-0040). - conn.execute(schema::CREATE_BLOBS_TABLE, ()) - .await - .map_err(storage_error)?; - // ADR-0047: idempotent migration that adds `expires_at` to pre-existing - // blobs tables. Duplicate-column errors are expected on newer deployments - // that already have the column from `CREATE_BLOBS_TABLE`; swallow them. - if let Err(error) = conn.execute(schema::ALTER_BLOBS_ADD_EXPIRES_AT, ()).await { - let message = error.to_string().to_ascii_lowercase(); - if !message.contains("duplicate column") { - return Err(storage_error(error)); - } - } - conn.execute(schema::CREATE_BLOBS_EXPIRES_AT_INDEX, ()) - .await - .map_err(storage_error)?; - - // Entity catalog — durable query-plane corpus. - conn.execute(schema::CREATE_ENTITY_CATALOG_TABLE, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_ENTITY_CATALOG_TYPE_INDEX, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_ENTITY_CATALOG_STATUS_INDEX, ()) - .await - .map_err(storage_error)?; - let _ = conn - .execute(schema::ALTER_ENTITY_CATALOG_ADD_PROJECTION_HASH, ()) - .await; - let _ = conn - .execute(schema::ALTER_ENTITY_CATALOG_ADD_FIELDS, ()) - .await; - let _ = conn - .execute(schema::ALTER_ENTITY_CATALOG_ADD_STATE, ()) - .await; - - // Entity field index — EAV table for OData filter push-down. - conn.execute(schema::CREATE_ENTITY_FIELD_INDEX_TABLE, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_ENTITY_FIELD_INDEX_LOOKUP, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_ENTITY_FIELD_INDEX_STATUS, ()) - .await - .map_err(storage_error)?; - - // Entity key index (ADR-0153) — declared composite-key -> entity_id, the - // negative-existence access path co-committed with the journal append. - conn.execute(schema::CREATE_ENTITY_KEY_INDEX_TABLE, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_ENTITY_KEY_INDEX_ENTITY, ()) - .await - .map_err(storage_error)?; - - // Entity vector index (ADR-0155) — declared vector paths for exact-scan kNN, - // maintained write-behind (the event append is followed by the index write). - conn.execute(schema::CREATE_ENTITY_VECTOR_INDEX_TABLE, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_ENTITY_VECTOR_INDEX_PARTITION, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_ENTITY_VECTOR_INDEX_ENTITY, ()) - .await - .map_err(storage_error)?; - conn.execute(schema::CREATE_VECTOR_INDEX_BACKFILL_WATERMARK, ()) - .await - .map_err(storage_error)?; - - Ok(()) + crate::migrations::migrate(&conn).await } - /// Obtain a connection handle to the underlying database. /// /// `Database::connect()` returns a lightweight handle, **not** a fresh TCP diff --git a/crates/temper-store-turso/src/store/ots.rs b/crates/temper-store-turso/src/store/ots.rs index 866758c68..b765ddb1c 100644 --- a/crates/temper-store-turso/src/store/ots.rs +++ b/crates/temper-store-turso/src/store/ots.rs @@ -7,6 +7,36 @@ use tracing::instrument; use super::TursoEventStore; use crate::metrics::TursoQueryTimer; +/// SQL used to persist a completed OTS trajectory. +pub(crate) const PERSIST_OTS_TRAJECTORY_SQL: &str = "INSERT INTO ots_trajectories \ + (trajectory_id, tenant, agent_id, session_id, outcome, turn_count, data, persistence_status, persist_attempts, last_error, created_at, updated_at) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, 'persisted', 0, NULL, datetime('now'), datetime('now')) \ + ON CONFLICT(trajectory_id) DO UPDATE SET \ + tenant = excluded.tenant, agent_id = excluded.agent_id, session_id = excluded.session_id, \ + outcome = excluded.outcome, entity_type = NULL, turn_count = excluded.turn_count, \ + data = excluded.data, persistence_status = excluded.persistence_status, \ + persist_attempts = excluded.persist_attempts, last_error = excluded.last_error, \ + created_at = excluded.created_at, updated_at = excluded.updated_at"; + +/// SQL used to enqueue or refresh an OTS trajectory for background persistence. +pub(crate) const ENQUEUE_OTS_TRAJECTORY_SQL: &str = "INSERT INTO ots_trajectories \ + (trajectory_id, tenant, agent_id, session_id, outcome, turn_count, data, persistence_status, persist_attempts, last_error, created_at, updated_at) \ + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, 'queued', 0, NULL, datetime('now'), datetime('now')) \ + ON CONFLICT(trajectory_id) DO UPDATE SET \ + tenant = excluded.tenant, agent_id = excluded.agent_id, session_id = excluded.session_id, \ + outcome = excluded.outcome, turn_count = excluded.turn_count, data = excluded.data, \ + persistence_status = 'queued', last_error = NULL, updated_at = datetime('now')"; + +/// SQL used to mark an OTS trajectory as durably persisted. +pub(crate) const MARK_OTS_TRAJECTORY_PERSISTED_SQL: &str = "UPDATE ots_trajectories \ + SET persistence_status = 'persisted', last_error = NULL, updated_at = datetime('now') \ + WHERE trajectory_id = ?1"; + +/// SQL used to mark an OTS trajectory as failed after a persistence attempt. +pub(crate) const MARK_OTS_TRAJECTORY_FAILED_SQL: &str = "UPDATE ots_trajectories \ + SET persistence_status = 'failed', persist_attempts = persist_attempts + 1, last_error = ?2, updated_at = datetime('now') \ + WHERE trajectory_id = ?1"; + /// Row returned by OTS trajectory list queries (metadata only, not full data). #[derive(Debug, Clone, serde::Serialize)] pub struct OtsTrajectoryRow { @@ -61,9 +91,7 @@ impl TursoEventStore { let _timer = TursoQueryTimer::start("turso.persist_ots_trajectory"); let conn = self.connection()?; conn.execute( - "INSERT OR REPLACE INTO ots_trajectories \ - (trajectory_id, tenant, agent_id, session_id, outcome, turn_count, data, persistence_status, persist_attempts, last_error, created_at, updated_at) \ - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, 'persisted', 0, NULL, datetime('now'), datetime('now'))", + PERSIST_OTS_TRAJECTORY_SQL, params![ p.trajectory_id.to_string(), p.tenant.to_string(), @@ -92,13 +120,7 @@ impl TursoEventStore { let _timer = TursoQueryTimer::start("turso.enqueue_ots_trajectory"); let conn = self.connection()?; conn.execute( - "INSERT INTO ots_trajectories \ - (trajectory_id, tenant, agent_id, session_id, outcome, turn_count, data, persistence_status, persist_attempts, last_error, created_at, updated_at) \ - VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, 'queued', 0, NULL, datetime('now'), datetime('now')) \ - ON CONFLICT(trajectory_id) DO UPDATE SET \ - tenant = excluded.tenant, agent_id = excluded.agent_id, session_id = excluded.session_id, \ - outcome = excluded.outcome, turn_count = excluded.turn_count, data = excluded.data, \ - persistence_status = 'queued', last_error = NULL, updated_at = datetime('now')", + ENQUEUE_OTS_TRAJECTORY_SQL, params![ p.trajectory_id.to_string(), p.tenant.to_string(), @@ -122,9 +144,7 @@ impl TursoEventStore { let _timer = TursoQueryTimer::start("turso.mark_ots_trajectory_persisted"); let conn = self.connection()?; conn.execute( - "UPDATE ots_trajectories \ - SET persistence_status = 'persisted', last_error = NULL, updated_at = datetime('now') \ - WHERE trajectory_id = ?1", + MARK_OTS_TRAJECTORY_PERSISTED_SQL, params![trajectory_id.to_string()], ) .await @@ -141,9 +161,7 @@ impl TursoEventStore { let _timer = TursoQueryTimer::start("turso.mark_ots_trajectory_failed"); let conn = self.connection()?; conn.execute( - "UPDATE ots_trajectories \ - SET persistence_status = 'failed', persist_attempts = persist_attempts + 1, last_error = ?2, updated_at = datetime('now') \ - WHERE trajectory_id = ?1", + MARK_OTS_TRAJECTORY_FAILED_SQL, params![trajectory_id.to_string(), error.to_string()], ) .await diff --git a/crates/temper-store-turso/tests/migration_ledger.rs b/crates/temper-store-turso/tests/migration_ledger.rs new file mode 100644 index 000000000..5ada313d7 --- /dev/null +++ b/crates/temper-store-turso/tests/migration_ledger.rs @@ -0,0 +1,43 @@ +use libsql::Builder; +use temper_store_turso::TursoEventStore; + +#[tokio::test] +async fn incompatible_schema_object_prevents_startup() { + let directory = tempfile::tempdir().expect("temporary database directory"); + let database_path = directory.path().join("incompatible-schema.db"); + + let database = Builder::new_local(&database_path) + .build() + .await + .expect("create legacy database"); + let connection = database.connect().expect("connect to legacy database"); + connection + .execute( + "CREATE VIEW tenant_installed_apps AS \ + SELECT 'tenant' AS tenant_id, 'app' AS app_name", + (), + ) + .await + .expect("install incompatible schema object"); + drop(connection); + drop(database); + + let url = format!("file:{}", database_path.display()); + let error = TursoEventStore::new(&url, None) + .await + .expect_err("an incompatible schema object must prevent store readiness"); + + let diagnostic = error.to_string(); + assert!( + diagnostic.contains("tenant_installed_apps"), + "diagnostic must name the incompatible capability: {diagnostic}" + ); + assert!( + diagnostic.contains("must be a table") && diagnostic.contains("found view"), + "diagnostic must explain the incompatible object kind: {diagnostic}" + ); + assert!( + diagnostic.contains("migration 4") && diagnostic.contains("apps-platform-and-secrets"), + "diagnostic must identify the incompatible migration: {diagnostic}" + ); +} diff --git a/docs/adrs/0180-versioned-turso-migration-ledger.md b/docs/adrs/0180-versioned-turso-migration-ledger.md new file mode 100644 index 000000000..0fcaafcf1 --- /dev/null +++ b/docs/adrs/0180-versioned-turso-migration-ledger.md @@ -0,0 +1,321 @@ +# ADR-0180: Versioned Turso migration ledger + +- Status: Proposed +- Date: 2026-07-13 +- Deciders: Temper core maintainers +- Related: + - ADR-0066: Storage stack backend selection + - ADR-0068: Turso write-gate retrospective + - ADR-0074: Turso-Postgres ETL methodology + - ARN-242: Turso startup schema changes swallow migration errors and have no version ledger + - `crates/temper-store-turso/src/store/mod.rs` + - `crates/temper-store-turso/src/router.rs` + +## Context + +`TursoEventStore::new` currently runs one ad hoc sequence of `CREATE` and +`ALTER` statements on every connection. Some failures propagate, some are +classified by matching error-message text, and several groups discard every +error. The sequence also creates `tenant_secrets` twice. Consequently, an +unsupported statement, conflicting schema object, permission failure, or +interrupted partial upgrade can be treated as a successful startup. The first +visible failure then occurs later on an unrelated data path, after the store has +already been admitted for traffic. + +The sequence records neither which changes committed nor which definition was +used. Two processes can also inspect and mutate the same unversioned schema at +the same time. `TenantStoreRouter` adds a second schema owner by creating its +platform tables after `TursoEventStore` reports ready. + +Temper needs a startup contract that can distinguish a fresh database, every +supported legacy shape, an interrupted upgrade, a changed historical migration, +and a database written by newer code. Readiness must mean that the complete +schema required by the current binary has been verified. + +## Decision + +### Sub-Decision 1: One append-only migration catalog owns all Turso schema + +The Turso store will define an ordered, contiguous catalog of immutable +migrations. The catalog includes the entity-store schema and the router's +platform tables, eliminating `TenantStoreRouter::migrate_platform` as a second +owner. + +The first catalog release groups the existing schema history into ordered +capability migrations: journal/snapshots, specs and integrations, +authorization/artifacts, installed apps and platform metadata, +trajectory/OTS extensions, blob/query-plane projections, and declared +key/vector indexes. Every future schema change appends a new version; released +migration definitions are never edited or reordered. + +**Why this approach**: a catalog makes upgrade order explicit without dropping +support for databases created by the former ad hoc sequence. Keeping all Turso +DDL behind the same runner gives every store instance the same readiness +contract. + +### Sub-Decision 2: Successful versions are recorded with content checksums + +The runner bootstraps one metadata table: + +```sql +CREATE TABLE IF NOT EXISTS temper_schema_migrations ( + version INTEGER PRIMARY KEY CHECK (version > 0), + name TEXT NOT NULL UNIQUE, + checksum TEXT NOT NULL CHECK (length(checksum) = 64), + applied_at TEXT NOT NULL DEFAULT (datetime('now')) +); +``` + +Each checksum is SHA-256 over the complete canonical migration definition: the +version, name, ordered operation kinds, object identifiers, SQL, conditional +application metadata, and every declared post-migration capability. Capability +manifests use an explicit versioned, length-prefixed field encoding over ordered +maps and sets; compiler debug formatting and serializer implementation details +never enter the durable checksum. A change to either mutation behavior or +compatibility validation therefore changes the checksum. Validator inventory +semantics carry dedicated policy labels: SQLite-equivalent index-owner discovery +is part of the owner-aware extra-index policy v2, so changing that behavior +rotates every affected prefix checksum. Startup validates the complete ledger +before applying work: + +- versions must be positive, unique, contiguous, and known to this binary; +- recorded names and checksums must exactly match the compiled catalog; +- a ledger version newer than the binary is an incompatible schema, not a + downgrade opportunity; +- a later recorded version with an earlier gap is corruption and prevents + readiness. + +Ledger DDL comparison is quote-aware and format-insensitive, so punctuation +spacing and comments do not make an otherwise identical ledger incompatible, +while SQL token boundaries and quoted literal values remain exact. + +**Why this approach**: version alone cannot detect a historical migration that +was edited in place. Checksums make the append-only rule executable. + +### Sub-Decision 3: Migration application and ledger insertion are atomic + +For each pending version, the runner starts an immediate transaction, rereads +that version's ledger row inside the transaction, applies the migration, verifies +the migration's schema capabilities, inserts exactly one ledger row, rereads and +validates that retained row, and commits. Any error rolls back both DDL and ledger +insertion. Ledger triggers are incompatible because the system ledger is the +durability boundary, not an extension point. Final readiness requires the exact +catalog length and head rather than accepting a valid prefix. Catalog-head +verification also runs inside an immediate libSQL transaction, so rollback-only +production-write probes remain on one pinned Hrana stream rather than allowing +autocommit writes to escape between standalone savepoint statements. The +in-transaction reread lets independent processes race safely: after one commits, +the next observes and validates the recorded checksum rather than replaying the +work. + +Local connections retain WAL and busy-timeout configuration before the runner +starts. The busy handler is installed before WAL initialization so concurrent +fresh opens can wait during journal-mode setup. Remote Turso connections use the +same libSQL transaction boundary but skip local-only PRAGMAs. + +**Why this approach**: SQLite/libSQL DDL is transactional. Coupling schema work +and its durable acknowledgement removes the partial-success state that the +current startup path permits. + +### Sub-Decision 4: Legacy reconciliation uses exact introspection + +Legacy databases have schema objects but no ledger. Catalog operations therefore +remain convergent: + +- tables and indexes use idempotent creation after confirming that an existing + object of the same name has the required object kind and declared semantics; +- a column is added only when `pragma_table_xinfo` proves it is absent, including + hidden generated columns whose declarations reuse a catalog column name; +- an existing required column is accepted without issuing `ALTER TABLE` only + when its declared type affinity, nullability, default expression, and primary + key ordinal match; +- unexpected columns are accepted only when they are visible, nullable, have no + default expression, are not part of the primary key, and do not shadow SQLite + row identifiers, which proves that existing runtime inserts can continue to + omit them without evaluating database code; +- unique keys and foreign keys are exact sets; unique-key capabilities retain + ordered columns, partial status, and normalized predicates so a partial index + cannot masquerade as the full uniqueness required by the runtime; predicate + keyword case, comments, and punctuation spacing are normalized without + changing quoted literals or token boundaries; +- table SQL restriction features (`CHECK`, column `COLLATE`, generated columns, + conflict and foreign-key deferral clauses, `AUTOINCREMENT`, `STRICT`, and + `WITHOUT ROWID`) must exactly match the catalog capability; named indexes must + match their owner, uniqueness, ordered key columns, collation/sort direction, + and partial predicate; additional indexes on catalog-owned tables are accepted + only when they are non-unique, non-partial, use visible columns rather than + expressions, and use built-in collations, so runtime writes do not execute + unverified predicates, expressions, or extension code; +- triggers are inventoried as normalized, owner-scoped capabilities; an + undeclared trigger on a catalog-owned table prevents readiness, except for + legacy OTS triggers that the existing rebuild contract deliberately preserves; + trigger owners are matched with SQLite's ASCII case-insensitive identifier + semantics and normalized before capability comparison, so alternate casing in + an `ON` clause cannot evade inventory, ledger protection, or OTS preservation; + preserved OTS triggers are admitted only under a parsed audit-sink contract: + an unconditional `AFTER INSERT` trigger with exactly one statement that inserts + `NEW.trajectory_id` into a one-column plain `TEXT` audit table; the sink must + have no foreign keys, table restrictions, secondary triggers, executable or + non-canonical indexes, or other columns; conditions, OTS mutations, multiple + statements, quoted or compound target expressions, and arbitrary side effects + prevent readiness because fixed example writes cannot prove arbitrary trigger + behavior; admitted audit triggers additionally pass rollback-only probes using + opaque database-generated identities and the same SQL as production fresh and + existing-row persisted upserts, queued inserts/conflict updates, and + failed/persisted status transitions on every startup; the persisted upsert uses + `ON CONFLICT DO UPDATE` rather than SQLite's delete-and-insert `REPLACE` + semantics, so admitted inbound `CASCADE` and `RESTRICT` references neither lose + child rows nor reject an existing-ID persist; each resulting row is verified, + and both probe rows and audit side effects are rolled back on the same pinned + transaction stream; +- schema inventory excludes only SQLite's literal reserved `sqlite_` prefix; + legal user objects such as `sqliteX...` remain subject to the same trigger and + index compatibility rules; +- the one legacy OTS shape that cannot add its non-constant timestamp default + in place is rebuilt only after exact pre-upgrade column validation; explicit + indexes and triggers are captured and recreated in the same transaction, while + unmodeled columns, unique keys, foreign keys, or table restrictions prevent + mutation; an already-updated OTS shape passes the same exact column and table + restriction validation before reconciliation is treated as complete; +- the legacy OTS primary-key index must have the canonical ordered columns, + ascending direction, and binary collation, and every legal user-table name is + scanned for inbound foreign keys before the old table can be dropped; an + already-current OTS table is not rejected for safe inbound references because + no destructive rebuild occurs; +- column discovery uses SQLite's extended table introspection so generated + columns are visible even when their declaration uses short-form `AS (...)`; +- every other DDL failure propagates with migration version, name, operation, + and object context. + +There is no duplicate-column error-message matching. A view where a table is +required, a malformed table, or unsupported SQL is an incompatible schema and +fails startup. + +**Why this approach**: exact schema state, not backend-specific prose in an error +message, determines whether an operation is already complete. + +### Sub-Decision 5: Readiness includes final capability verification + +After every migration and again at the full-catalog head, the runner checks all +required object kinds; column affinity, nullability, defaults, and primary-key +positions; omission safety of unexpected columns; exact unique/foreign-key and +table-restriction semantics; named-index owners, uniqueness, key ordering, +collation/sort direction, and predicates; trigger ownership and normalized +definitions; executable extension policy; and the ledger head. These capability +declarations are part of the migration checksum. Each pending migration verifies +its cumulative prefix before its ledger row commits. Final readiness verifies +only the current catalog-head snapshot: later append-only migrations are allowed +to evolve capabilities introduced by earlier versions without making those +historical prefix snapshots incompatible with the declared head. A store is returned from +`TursoEventStore::new` only after that verification succeeds. The final head +check uses a real transaction on local SQLite and remote Hrana, not a standalone +savepoint on an autocommit connection. Diagnostics +identify the catalog head and the missing or incompatible capability so +operators can repair or restore the database without waiting for a later query +to fail. + +**Why this approach**: a successful `CREATE TABLE IF NOT EXISTS` says only that +an object name exists. It does not prove that an old or manually modified object +can serve the current data paths. + +## Rollout Plan + +1. Ship the catalog, ledger, transactional runner, exact legacy reconciliation, + router-schema consolidation, and the complete regression matrix together. +2. On first startup, fresh databases apply the catalog in order; legacy + databases reconcile their existing objects and atomically record each version. +3. Refuse traffic on checksum mismatch, a newer ledger head, malformed legacy + objects, or any unexpected DDL/verification failure. +4. Verify the exact startup flow locally with concurrent independent store + instances and retain the commands/output on the pull request. + +## Readiness Gates + +- Fresh install records every catalog version and passes final verification. +- Every catalog prefix upgrades to the current head and a second startup is a + no-op. +- A legacy no-ledger database and representative partial legacy schemas converge. +- Restrictive extra columns, unique keys, foreign keys, and table modifiers fail + before their owning version is recorded, while nullable column extensions stay + compatible with canonical runtime inserts. +- Undeclared triggers and executable expression/partial indexes fail before + readiness and remain unmodified; plain non-unique indexes and structurally + verified OTS audit triggers retain their supported behavior. +- A later migration can tighten a table introduced by an earlier version, + commit its ledger row, and pass replay against the declared catalog head. +- Injected DDL/permission failure rolls back the active version and prevents + `TursoEventStore::new` from returning a store. +- A checksum mismatch, ledger gap, or newer schema version prevents readiness + with an actionable diagnostic. +- Concurrent independent startups produce one valid, contiguous ledger. +- A structurally verified legacy OTS audit trigger on remote Hrana leaves no + durable probe or audit rows; recognizable-input bypasses and unmodeled trigger + mutations prevent readiness; current inbound `CASCADE`/`RESTRICT` references + survive an existing-ID production persist. +- Existing Turso event-store behavior remains green across the workspace. + +## Consequences + +### Positive + +- Startup fails at the schema boundary instead of accepting mixed schemas. +- Operators can identify the exact migration catalog applied to a database. +- Legacy adoption, retries, and concurrent startup have one durable contract. +- Platform/router tables no longer have a separate migration path. + +### Negative + +- Startup performs bounded schema introspection and ledger validation. +- Historical migration definitions become immutable; correcting one requires a + new compensating migration. +- An older binary cannot open a database whose ledger was advanced by newer code. +- Legacy OTS triggers outside the explicit audit-sink contract require operator + removal or a future catalog-declared integration before the database can become + ready. + +### Risks + +- An incomplete capability manifest could admit a malformed legacy table. + Mitigation: each migration declares and tests its required objects/columns, + followed by a full-catalog verification. +- Remote libSQL transaction behavior could diverge from local SQLite. + Mitigation: use only the transaction and introspection surfaces exposed by the + shared `libsql` API and retain remote-compatible SQL. +- Long-running concurrent startup could exhaust the configured busy timeout. + Mitigation: migrations are bounded, transactions cover one version at a time, + and lock errors fail readiness rather than being swallowed. + +### DST Compliance + +This decision changes only `temper-store-turso` startup persistence code. It does +not touch the simulation-visible `temper-runtime`, `temper-jit`, or +`temper-server` crates. The concurrency regression uses independent database +connections and durable ledger assertions; no simulation exception is required. + +## Non-Goals + +- Changing the Postgres migration system or Redis storage. +- Redesigning domain tables or backfilling domain data. +- Allowing automatic downgrade or checksum repair. +- Keeping the former best-effort startup path as a compatibility fallback. + +## Alternatives Considered + +1. **Keep the current sequence and classify more errors** — rejected. Error text + is not a durable migration record, cannot detect edited history/newer schemas, + and still permits partial upgrades. +2. **Use only `PRAGMA user_version`** — rejected. It stores one integer with no + per-migration checksum, diagnostic history, or atomic proof for each change. +3. **Adopt an external migration framework** — rejected for this bounded backend. + The required ordering, checksums, introspection, and transactions fit behind a + small store-local runner without adding a second database abstraction. +4. **Baseline every existing database as current** — rejected. That would record + success without proving missing columns/indexes and preserve the original bug. + +## Rollback Policy + +Do not delete or rewrite ledger rows. Before production migration, rollback is a +normal binary rollback. After a database records a version unknown to the older +binary, restore a pre-migration database snapshot or deploy a forward +compensating migration with a compatible binary. Domain data is never discarded +to force a downgrade.