fix(store-turso): fail-closed migrations with schema ledger (ARN-242)

crates/temper-store-turso/src/schema.rs

@@ -1,11 +1,17 @@
 //! SQLite-compatible schema for the Turso/libSQL event store.
 
+mod migrations;
 mod query_plane;
+mod trajectories;
 
 pub use crate::schema_event_history::{
     ALTER_EVENTS_ADD_SEGMENT_INDEX, CREATE_EVENT_SEGMENTS_OPEN_INDEX, CREATE_EVENT_SEGMENTS_TABLE,
     CREATE_SNAPSHOT_HISTORY_ENTITY_INDEX, CREATE_SNAPSHOT_HISTORY_TABLE,
 };
+pub use migrations::{
+    ALTER_SCHEMA_MIGRATIONS_ADD_FINGERPRINT, CREATE_SCHEMA_MIGRATIONS_TABLE, INSERT_SCHEMA_VERSION,
+    SELECT_SCHEMA_FINGERPRINT_APPLIED,
+};
 pub use query_plane::{
     CREATE_ENTITY_CATALOG_STATUS_INDEX, CREATE_ENTITY_CATALOG_TABLE,
     CREATE_ENTITY_CATALOG_TYPE_INDEX, CREATE_ENTITY_FIELD_INDEX_LOOKUP,
@@ -14,6 +20,20 @@ pub use query_plane::{
     CREATE_ENTITY_VECTOR_INDEX_ENTITY, CREATE_ENTITY_VECTOR_INDEX_PARTITION,
     CREATE_ENTITY_VECTOR_INDEX_TABLE, CREATE_VECTOR_INDEX_BACKFILL_WATERMARK,
 };
+pub use trajectories::{
+    ALTER_OTS_TRAJECTORIES_ADD_LAST_ERROR, ALTER_OTS_TRAJECTORIES_ADD_PERSIST_ATTEMPTS,
+    ALTER_OTS_TRAJECTORIES_ADD_PERSISTENCE_STATUS, ALTER_OTS_TRAJECTORIES_ADD_UPDATED_AT,
+    ALTER_TRAJECTORIES_ADD_AGENT_ID, ALTER_TRAJECTORIES_ADD_AUTHZ_DENIED,
+    ALTER_TRAJECTORIES_ADD_DENIED_MODULE, ALTER_TRAJECTORIES_ADD_DENIED_RESOURCE,
+    ALTER_TRAJECTORIES_ADD_INTENT, ALTER_TRAJECTORIES_ADD_MATCHED_POLICY_IDS,
+    ALTER_TRAJECTORIES_ADD_REQUEST_BODY, ALTER_TRAJECTORIES_ADD_SESSION_ID,
+    ALTER_TRAJECTORIES_ADD_SOURCE, ALTER_TRAJECTORIES_ADD_SPEC_GOVERNED,
+    CREATE_OTS_TRAJECTORIES_AGENT_INDEX, CREATE_OTS_TRAJECTORIES_OUTCOME_INDEX,
+    CREATE_OTS_TRAJECTORIES_STATUS_INDEX, CREATE_OTS_TRAJECTORIES_TABLE,
+    CREATE_OTS_TRAJECTORIES_TENANT_INDEX, CREATE_TRAJECTORIES_AGENT_INDEX,
+    CREATE_TRAJECTORIES_ENTITY_ACTION_INDEX, CREATE_TRAJECTORIES_SUCCESS_INDEX,
+    CREATE_TRAJECTORIES_TABLE,
+};
 
 pub const CREATE_EVENTS_TABLE: &str = "\
 CREATE TABLE IF NOT EXISTS events (
@@ -61,28 +81,6 @@ CREATE TABLE IF NOT EXISTS specs (
     UNIQUE(tenant, entity_type)
 );";
 
-pub const CREATE_TRAJECTORIES_TABLE: &str = "\
-CREATE TABLE IF NOT EXISTS trajectories (
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    tenant TEXT NOT NULL,
-    entity_type TEXT NOT NULL,
-    entity_id TEXT NOT NULL,
-    action TEXT NOT NULL,
-    success INTEGER NOT NULL DEFAULT 0,
-    from_status TEXT,
-    to_status TEXT,
-    error TEXT,
-    created_at TEXT NOT NULL DEFAULT (datetime('now'))
-);";
-
-pub const CREATE_TRAJECTORIES_SUCCESS_INDEX: &str = "\
-CREATE INDEX IF NOT EXISTS idx_trajectories_success
-    ON trajectories(success);";
-
-pub const CREATE_TRAJECTORIES_ENTITY_ACTION_INDEX: &str = "\
-CREATE INDEX IF NOT EXISTS idx_trajectories_entity_action
-    ON trajectories(tenant, entity_type, action);";
-
 pub const CREATE_TENANT_CONSTRAINTS_TABLE: &str = "\
 CREATE TABLE IF NOT EXISTS tenant_constraints (
     tenant TEXT NOT NULL PRIMARY KEY,
@@ -277,35 +275,6 @@ pub const ALTER_SPECS_ADD_CONTENT_HASH: &str = "ALTER TABLE specs ADD COLUMN con
 pub const ALTER_SPECS_ADD_COMMITTED: &str =
     "ALTER TABLE specs ADD COLUMN committed INTEGER NOT NULL DEFAULT 1";
 
-/// ALTER TABLE migrations for the `trajectories` table.
-///
-/// These add columns that were previously only tracked in-memory
-/// (agent_id, session_id, authz_denied, etc.). Each statement uses
-/// try-and-ignore semantics in SQLite (duplicate column is a no-op error).
-pub const ALTER_TRAJECTORIES_ADD_AGENT_ID: &str =
-    "ALTER TABLE trajectories ADD COLUMN agent_id TEXT";
-pub const ALTER_TRAJECTORIES_ADD_SESSION_ID: &str =
-    "ALTER TABLE trajectories ADD COLUMN session_id TEXT";
-pub const ALTER_TRAJECTORIES_ADD_AUTHZ_DENIED: &str =
-    "ALTER TABLE trajectories ADD COLUMN authz_denied INTEGER";
-pub const ALTER_TRAJECTORIES_ADD_DENIED_RESOURCE: &str =
-    "ALTER TABLE trajectories ADD COLUMN denied_resource TEXT";
-pub const ALTER_TRAJECTORIES_ADD_DENIED_MODULE: &str =
-    "ALTER TABLE trajectories ADD COLUMN denied_module TEXT";
-pub const ALTER_TRAJECTORIES_ADD_SOURCE: &str = "ALTER TABLE trajectories ADD COLUMN source TEXT";
-pub const ALTER_TRAJECTORIES_ADD_SPEC_GOVERNED: &str =
-    "ALTER TABLE trajectories ADD COLUMN spec_governed INTEGER";
-pub const ALTER_TRAJECTORIES_ADD_REQUEST_BODY: &str =
-    "ALTER TABLE trajectories ADD COLUMN request_body TEXT";
-pub const ALTER_TRAJECTORIES_ADD_INTENT: &str = "ALTER TABLE trajectories ADD COLUMN intent TEXT";
-pub const ALTER_TRAJECTORIES_ADD_MATCHED_POLICY_IDS: &str =
-    "ALTER TABLE trajectories ADD COLUMN matched_policy_ids TEXT";
-
-/// Index on agent_id for agent-scoped trajectory queries.
-pub const CREATE_TRAJECTORIES_AGENT_INDEX: &str = "\
-CREATE INDEX IF NOT EXISTS idx_trajectories_agent
-    ON trajectories(agent_id);";
-
 /// Feature request records generated from trajectory analysis.
 pub const CREATE_FEATURE_REQUESTS_TABLE: &str = "\
 CREATE TABLE IF NOT EXISTS feature_requests (
@@ -443,56 +412,6 @@ CREATE INDEX IF NOT EXISTS idx_blobs_expires_at ON blobs(expires_at) WHERE expir
 // OTS trajectory storage (full agent execution traces)
 // ---------------------------------------------------------------------------
 
-/// Full OTS trajectory storage for GEPA self-improvement loop.
-///
-/// Stores complete agent execution traces (tool calls, decisions, reasoning)
-/// captured by the MCP server during agent sessions. The `data` column holds
-/// the full OTS JSON blob; indexed columns enable efficient filtering.
-pub const CREATE_OTS_TRAJECTORIES_TABLE: &str = "\
-CREATE TABLE IF NOT EXISTS ots_trajectories (
-    trajectory_id TEXT PRIMARY KEY,
-    tenant TEXT NOT NULL,
-    agent_id TEXT NOT NULL,
-    session_id TEXT,
-    outcome TEXT NOT NULL DEFAULT 'unknown',
-    entity_type TEXT,
-    turn_count INTEGER NOT NULL DEFAULT 0,
-    data TEXT NOT NULL,
-    persistence_status TEXT NOT NULL DEFAULT 'persisted',
-    persist_attempts INTEGER NOT NULL DEFAULT 0,
-    last_error TEXT,
-    created_at TEXT NOT NULL DEFAULT (datetime('now')),
-    updated_at TEXT NOT NULL DEFAULT (datetime('now'))
-);";
-
-pub const ALTER_OTS_TRAJECTORIES_ADD_PERSISTENCE_STATUS: &str = "\
-ALTER TABLE ots_trajectories ADD COLUMN persistence_status TEXT NOT NULL DEFAULT 'persisted';";
-
-pub const ALTER_OTS_TRAJECTORIES_ADD_PERSIST_ATTEMPTS: &str = "\
-ALTER TABLE ots_trajectories ADD COLUMN persist_attempts INTEGER NOT NULL DEFAULT 0;";
-
-pub const ALTER_OTS_TRAJECTORIES_ADD_LAST_ERROR: &str = "\
-ALTER TABLE ots_trajectories ADD COLUMN last_error TEXT;";
-
-pub const ALTER_OTS_TRAJECTORIES_ADD_UPDATED_AT: &str = "\
-ALTER TABLE ots_trajectories ADD COLUMN updated_at TEXT NOT NULL DEFAULT (datetime('now'));";
-
-pub const CREATE_OTS_TRAJECTORIES_AGENT_INDEX: &str = "\
-CREATE INDEX IF NOT EXISTS idx_ots_trajectories_agent
-    ON ots_trajectories(agent_id);";
-
-pub const CREATE_OTS_TRAJECTORIES_TENANT_INDEX: &str = "\
-CREATE INDEX IF NOT EXISTS idx_ots_trajectories_tenant
-    ON ots_trajectories(tenant);";
-
-pub const CREATE_OTS_TRAJECTORIES_OUTCOME_INDEX: &str = "\
-CREATE INDEX IF NOT EXISTS idx_ots_trajectories_outcome
-    ON ots_trajectories(outcome);";
-
-pub const CREATE_OTS_TRAJECTORIES_STATUS_INDEX: &str = "\
-CREATE INDEX IF NOT EXISTS idx_ots_trajectories_status
-    ON ots_trajectories(persistence_status, updated_at);";
-
 #[cfg(test)]
 #[path = "schema_test.rs"]
 mod schema_test;

crates/temper-store-turso/src/schema/migrations.rs

@@ -0,0 +1,57 @@
+//! ARN-242: the schema version ledger.
+
+/// Every successful `migrate()` run records the version it brought the
+/// database to; boots short-circuit when the ledger already shows the current
+/// version. EVERY schema change must bump `SCHEMA_VERSION` in `store/mod.rs`,
+/// or stamped databases will skip it.
+pub const CREATE_SCHEMA_MIGRATIONS_TABLE: &str = "\
+CREATE TABLE IF NOT EXISTS temper_schema_migrations (
+    version INTEGER PRIMARY KEY,
+    name TEXT NOT NULL,
+    fingerprint TEXT NOT NULL DEFAULT '',
+    applied_at TEXT NOT NULL
+)";
+
+/// The ledger's OWN migration. The ledger table sits BEFORE the gate that
+/// governs everything else, so it can never be gated by its own fingerprint —
+/// it must migrate itself, un-gated, on every boot. Without this, adding any
+/// column to the ledger (`fingerprint` here; `applied_by`, `duration_ms`,
+/// anything later) makes the very next statement — the gate SELECT — fail at
+/// prepare time with "no such column" on every database whose ledger predates
+/// it: a hard boot failure, and precisely the class this ADR exists to kill.
+///
+/// Routed through `execute_idempotent`, so on a fresh database (where
+/// `CREATE TABLE` already declared the column) it is a tolerated
+/// duplicate-column no-op that leaves `sqlite_master` — and therefore
+/// `SCHEMA_FINGERPRINT` — unchanged.
+pub const ALTER_SCHEMA_MIGRATIONS_ADD_FINGERPRINT: &str = "\
+ALTER TABLE temper_schema_migrations ADD COLUMN fingerprint TEXT NOT NULL DEFAULT ''";
+
+/// Has this database EVER been migrated to the declared schema? This — not
+/// the version — is the boot gate: the DDL is skipped only when a ledger row
+/// records the fingerprint the binary declares, so a schema change cannot skip
+/// existing databases even if the author forgets to bump `SCHEMA_VERSION`.
+///
+/// Asking "ever migrated to this schema" rather than "is the LATEST row this
+/// schema" also makes a rollback cheap: a binary rolled back to an older
+/// schema finds its own retained row and skips, instead of re-running the
+/// whole DDL on every boot for as long as the rollback lasts. (Caveat: when
+/// the newer build changed the schema WITHOUT bumping the version — the
+/// version is only a label — `INSERT OR REPLACE` overwrote the older row, so
+/// the rolled-back binary re-runs the DDL once and then skips.)
+///
+/// It compares the STORED declared constant against the CURRENT declared
+/// constant — never the live schema — so a platform database's extra
+/// `migrate_platform` tables cannot cause spurious re-runs.
+pub const SELECT_SCHEMA_FINGERPRINT_APPLIED: &str = "\
+SELECT EXISTS(SELECT 1 FROM temper_schema_migrations WHERE fingerprint = ?1)";
+
+/// Stamp a successfully applied schema version and its fingerprint.
+/// `INSERT OR REPLACE` on the version primary key makes a concurrent
+/// double-stamp idempotent, and lets a same-version DDL change (an author who
+/// updated the fingerprint without bumping the version) record its new
+/// fingerprint after the DDL actually ran.
+pub const INSERT_SCHEMA_VERSION: &str = "\
+INSERT OR REPLACE INTO temper_schema_migrations \
+(version, name, fingerprint, applied_at) \
+VALUES (?1, ?2, ?3, datetime('now'))";

crates/temper-store-turso/src/schema/trajectories.rs

@@ -0,0 +1,111 @@
+//! Trajectory and OTS-trajectory table schema.
+
+pub const CREATE_TRAJECTORIES_TABLE: &str = "\
+CREATE TABLE IF NOT EXISTS trajectories (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    tenant TEXT NOT NULL,
+    entity_type TEXT NOT NULL,
+    entity_id TEXT NOT NULL,
+    action TEXT NOT NULL,
+    success INTEGER NOT NULL DEFAULT 0,
+    from_status TEXT,
+    to_status TEXT,
+    error TEXT,
+    created_at TEXT NOT NULL DEFAULT (datetime('now'))
+);";
+
+pub const CREATE_TRAJECTORIES_SUCCESS_INDEX: &str = "\
+CREATE INDEX IF NOT EXISTS idx_trajectories_success
+    ON trajectories(success);";
+
+pub const CREATE_TRAJECTORIES_ENTITY_ACTION_INDEX: &str = "\
+CREATE INDEX IF NOT EXISTS idx_trajectories_entity_action
+    ON trajectories(tenant, entity_type, action);";
+
+/// ALTER TABLE migrations for the `trajectories` table.
+///
+/// These add columns that were previously only tracked in-memory
+/// (agent_id, session_id, authz_denied, etc.). Each statement uses
+/// try-and-ignore semantics in SQLite (duplicate column is a no-op error).
+pub const ALTER_TRAJECTORIES_ADD_AGENT_ID: &str =
+    "ALTER TABLE trajectories ADD COLUMN agent_id TEXT";
+
+pub const ALTER_TRAJECTORIES_ADD_SESSION_ID: &str =
+    "ALTER TABLE trajectories ADD COLUMN session_id TEXT";
+
+pub const ALTER_TRAJECTORIES_ADD_AUTHZ_DENIED: &str =
+    "ALTER TABLE trajectories ADD COLUMN authz_denied INTEGER";
+
+pub const ALTER_TRAJECTORIES_ADD_DENIED_RESOURCE: &str =
+    "ALTER TABLE trajectories ADD COLUMN denied_resource TEXT";
+
+pub const ALTER_TRAJECTORIES_ADD_DENIED_MODULE: &str =
+    "ALTER TABLE trajectories ADD COLUMN denied_module TEXT";
+
+pub const ALTER_TRAJECTORIES_ADD_SOURCE: &str = "ALTER TABLE trajectories ADD COLUMN source TEXT";
+
+pub const ALTER_TRAJECTORIES_ADD_SPEC_GOVERNED: &str =
+    "ALTER TABLE trajectories ADD COLUMN spec_governed INTEGER";
+
+pub const ALTER_TRAJECTORIES_ADD_REQUEST_BODY: &str =
+    "ALTER TABLE trajectories ADD COLUMN request_body TEXT";
+
+pub const ALTER_TRAJECTORIES_ADD_INTENT: &str = "ALTER TABLE trajectories ADD COLUMN intent TEXT";
+
+pub const ALTER_TRAJECTORIES_ADD_MATCHED_POLICY_IDS: &str =
+    "ALTER TABLE trajectories ADD COLUMN matched_policy_ids TEXT";
+
+/// Index on agent_id for agent-scoped trajectory queries.
+pub const CREATE_TRAJECTORIES_AGENT_INDEX: &str = "\
+CREATE INDEX IF NOT EXISTS idx_trajectories_agent
+    ON trajectories(agent_id);";
+
+/// Full OTS trajectory storage for GEPA self-improvement loop.
+///
+/// Stores complete agent execution traces (tool calls, decisions, reasoning)
+/// captured by the MCP server during agent sessions. The `data` column holds
+/// the full OTS JSON blob; indexed columns enable efficient filtering.
+pub const CREATE_OTS_TRAJECTORIES_TABLE: &str = "\
+CREATE TABLE IF NOT EXISTS ots_trajectories (
+    trajectory_id TEXT PRIMARY KEY,
+    tenant TEXT NOT NULL,
+    agent_id TEXT NOT NULL,
+    session_id TEXT,
+    outcome TEXT NOT NULL DEFAULT 'unknown',
+    entity_type TEXT,
+    turn_count INTEGER NOT NULL DEFAULT 0,
+    data TEXT NOT NULL,
+    persistence_status TEXT NOT NULL DEFAULT 'persisted',
+    persist_attempts INTEGER NOT NULL DEFAULT 0,
+    last_error TEXT,
+    created_at TEXT NOT NULL DEFAULT (datetime('now')),
+    updated_at TEXT NOT NULL DEFAULT (datetime('now'))
+);";
+
+pub const ALTER_OTS_TRAJECTORIES_ADD_PERSISTENCE_STATUS: &str = "\
+ALTER TABLE ots_trajectories ADD COLUMN persistence_status TEXT NOT NULL DEFAULT 'persisted';";
+
+pub const ALTER_OTS_TRAJECTORIES_ADD_PERSIST_ATTEMPTS: &str = "\
+ALTER TABLE ots_trajectories ADD COLUMN persist_attempts INTEGER NOT NULL DEFAULT 0;";
+
+pub const ALTER_OTS_TRAJECTORIES_ADD_LAST_ERROR: &str = "\
+ALTER TABLE ots_trajectories ADD COLUMN last_error TEXT;";
+
+pub const ALTER_OTS_TRAJECTORIES_ADD_UPDATED_AT: &str = "\
+ALTER TABLE ots_trajectories ADD COLUMN updated_at TEXT NOT NULL DEFAULT (datetime('now'));";
+
+pub const CREATE_OTS_TRAJECTORIES_AGENT_INDEX: &str = "\
+CREATE INDEX IF NOT EXISTS idx_ots_trajectories_agent
+    ON ots_trajectories(agent_id);";
+
+pub const CREATE_OTS_TRAJECTORIES_TENANT_INDEX: &str = "\
+CREATE INDEX IF NOT EXISTS idx_ots_trajectories_tenant
+    ON ots_trajectories(tenant);";
+
+pub const CREATE_OTS_TRAJECTORIES_OUTCOME_INDEX: &str = "\
+CREATE INDEX IF NOT EXISTS idx_ots_trajectories_outcome
+    ON ots_trajectories(outcome);";
+
+pub const CREATE_OTS_TRAJECTORIES_STATUS_INDEX: &str = "\
+CREATE INDEX IF NOT EXISTS idx_ots_trajectories_status
+    ON ots_trajectories(persistence_status, updated_at);";