diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9ba6bc7f8d..8d0fbad43c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -259,6 +259,54 @@ jobs: - name: Run Content DB tests run: pnpm test:content-db + - name: Install isolated PGlite test driver + run: >- + npm install --ignore-scripts --no-save + --prefix "$RUNNER_TEMP/s2573-pglite" + @electric-sql/pglite@0.5.4 + + - name: Run PGlite lock and migration action semantics tests + run: node scripts/test-content-database-lock-pglite.mjs + env: + S2573_PGLITE_INSTALL_PREFIX: ${{ runner.temp }}/s2573-pglite + + content-db-postgres-tests: + name: Content DB PostgreSQL locking + runs-on: ubuntu-latest + timeout-minutes: 20 + services: + postgres: + image: postgres:17-alpine + env: + POSTGRES_DB: content_migration_test + POSTGRES_HOST_AUTH_METHOD: trust + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres -d content_migration_test" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1 + + - uses: pnpm/action-setup@fc06bc1257f339d1d5d8b3a19a8cae5388b55320 # v4.4.0 + + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: "22" + cache: "pnpm" + + - name: Restore dist + tsBuildInfo cache + uses: ./.github/actions/restore-dist-cache + + - run: pnpm install --frozen-lockfile + + - name: Run Content PostgreSQL lock integration tests + run: pnpm test:content-db-postgres + env: + CONTENT_MIGRATION_POSTGRES_URL: postgres://postgres@127.0.0.1:5432/content_migration_test + core-integration-tests: name: Core integration tests runs-on: ubuntu-latest diff --git a/package.json b/package.json index 82a247866f..0f61c2ecac 100644 --- a/package.json +++ b/package.json @@ -30,7 +30,8 @@ "fix:imports": "oxfmt --write .", "test": "tsx scripts/workspace-run.ts test", "test:fast": "tsx scripts/workspace-run.ts test -- --exclude \"**/*.db.test.ts\" --exclude \"**/*.integration.spec.ts\" --exclude \"**/*.integration.test.ts\" --exclude \"**/*.e2e.spec.ts\" --exclude \"**/*.e2e.test.ts\" --exclude \"**/e2e/**\" --exclude \"**/*.live.spec.ts\" --exclude \"**/*.live.test.ts\" --exclude \"**/*.perf.spec.ts\" --exclude \"**/*.perf.test.ts\" --exclude \"**/create-e2e.spec.ts\"", - "test:content-db": "pnpm --filter content exec vitest --run actions/bind-content-database-source-field.db.test.ts actions/blocks-seeding.db.test.ts actions/builder-source-review-gates.db.test.ts actions/content-database-lifecycle.db.test.ts actions/database-row-batch-actions.db.test.ts actions/list-content-databases.db.test.ts actions/move-document.db.test.ts actions/resync-content-database-source.db.test.ts actions/slack-correction-identity.db.test.ts actions/stage-builder-source-bulk-update.db.test.ts actions/submit-content-database-form.db.test.ts actions/update-document.db.test.ts --config vitest.config.ts", + "test:content-db": "pnpm --filter content exec vitest --run actions/bind-content-database-source-field.db.test.ts actions/blocks-seeding.db.test.ts actions/builder-source-review-gates.db.test.ts actions/content-database-lifecycle.db.test.ts actions/database-row-batch-actions.db.test.ts actions/list-content-databases.db.test.ts actions/migrate-content-database-rows.db.test.ts actions/move-document.db.test.ts actions/resync-content-database-source.db.test.ts actions/slack-correction-identity.db.test.ts actions/stage-builder-source-bulk-update.db.test.ts actions/submit-content-database-form.db.test.ts actions/update-document.db.test.ts --config vitest.config.ts", + "test:content-db-postgres": "pnpm --filter content exec vitest --run actions/migrate-content-database-rows.postgres.integration.test.ts --config vitest.config.ts", "test:core-integration": "pnpm --filter @agent-native/core exec vitest --run src/agent/engine/translate-ai-sdk.integration.spec.ts src/agent/run-loop-with-resume.integration.spec.ts src/client/extensions/AgentNativeExtensionFrame.e2e.spec.ts src/client/session-replay-iframe.e2e.spec.ts src/scripts/db/migrate-encrypt-credentials.e2e.spec.ts src/scripts/db/scope-isolation.e2e.spec.ts src/server/csrf-plugin-ordering.integration.spec.ts src/server/embedded.integration.spec.ts --passWithNoTests", "test:plan-e2e": "pnpm --filter plan exec vitest --run actions/create-visual-recap.e2e.spec.ts --passWithNoTests", "test:trusted-acceptance": "tsx --test scripts/trusted-acceptance.spec.ts scripts/guard-trusted-acceptance-workflow.spec.ts scripts/trusted-acceptance/*.spec.ts", diff --git a/scripts/test-content-database-lock-pglite.mjs b/scripts/test-content-database-lock-pglite.mjs new file mode 100644 index 0000000000..147f3223db --- /dev/null +++ b/scripts/test-content-database-lock-pglite.mjs @@ -0,0 +1,181 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, rmSync, rmdirSync, symlinkSync } from "node:fs"; +import { createRequire } from "node:module"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + +const installPrefix = process.env.S2573_PGLITE_INSTALL_PREFIX; +if (!installPrefix) { + throw new Error("S2573_PGLITE_INSTALL_PREFIX is required."); +} + +const requireFromFixture = createRequire(join(installPrefix, "package.json")); +const entry = requireFromFixture.resolve("@electric-sql/pglite"); +const { PGlite } = await import(pathToFileURL(entry).href); +const client = await PGlite.create("memory://"); + +try { + await client.exec(` + CREATE TABLE content_databases ( + id TEXT PRIMARY KEY, + updated_at TEXT NOT NULL + ); + CREATE TABLE content_database_items ( + id TEXT PRIMARY KEY, + database_id TEXT NOT NULL, + updated_at TEXT NOT NULL + ); + CREATE TABLE documents ( + id TEXT PRIMARY KEY, + content TEXT NOT NULL + ); + INSERT INTO content_databases (id, updated_at) + VALUES ('synthetic_pglite_database', '2026-01-01T00:00:00.000Z'); + INSERT INTO content_database_items (id, database_id, updated_at) + VALUES ( + 'synthetic_pglite_membership', + 'synthetic_pglite_database', + '2026-01-01T00:00:00.000Z' + ); + INSERT INTO documents (id, content) + VALUES ('synthetic_pglite_document', '# Before'); + `); + + const databaseLock = await client.query(` + UPDATE content_databases + SET updated_at = updated_at + WHERE id = 'synthetic_pglite_database' + RETURNING id, updated_at + `); + const membershipLock = await client.query(` + UPDATE content_database_items + SET updated_at = updated_at + WHERE id = 'synthetic_pglite_membership' + RETURNING id, updated_at + `); + assert.deepEqual(databaseLock.rows, [ + { + id: "synthetic_pglite_database", + updated_at: "2026-01-01T00:00:00.000Z", + }, + ]); + assert.deepEqual(membershipLock.rows, [ + { + id: "synthetic_pglite_membership", + updated_at: "2026-01-01T00:00:00.000Z", + }, + ]); + + await client.query(` + UPDATE content_databases + SET updated_at = '2026-01-02T00:00:00.000Z' + WHERE id = 'synthetic_pglite_database' + `); + const touched = await client.query(` + SELECT updated_at + FROM content_databases + WHERE id = 'synthetic_pglite_database' + `); + assert.equal(touched.rows[0]?.updated_at, "2026-01-02T00:00:00.000Z"); + + let releaseTransaction = () => {}; + const transactionReleased = new Promise((resolve) => { + releaseTransaction = resolve; + }); + let lockAcquired = () => {}; + const transactionLocked = new Promise((resolve) => { + lockAcquired = resolve; + }); + const transaction = client.transaction(async (tx) => { + await tx.query(` + UPDATE content_databases + SET updated_at = updated_at + WHERE id = 'synthetic_pglite_database' + `); + lockAcquired(); + await transactionReleased; + }); + await transactionLocked; + + let editorWriteFinished = false; + const editorWrite = client + .query(` + UPDATE documents + SET content = '# Saved editor body' + WHERE id = 'synthetic_pglite_document' + `) + .then(() => { + editorWriteFinished = true; + }); + await new Promise((resolve) => setTimeout(resolve, 100)); + assert.equal(editorWriteFinished, false); + + releaseTransaction(); + await transaction; + await editorWrite; + assert.equal(editorWriteFinished, true); +} finally { + await client.close(); +} + +const driverPackage = join( + installPrefix, + "node_modules", + "@electric-sql", + "pglite", +); +const driverScopes = [ + join(process.cwd(), "node_modules", "@electric-sql"), + join(process.cwd(), "packages", "core", "node_modules", "@electric-sql"), +]; +const driverLinks = driverScopes.map((scopeDirectory) => ({ + scopeDirectory, + scopeExisted: existsSync(scopeDirectory), + driverLink: join(scopeDirectory, "pglite"), +})); +for (const { driverLink } of driverLinks) { + if (existsSync(driverLink)) { + throw new Error( + `Refusing to replace existing PGlite driver at ${driverLink}`, + ); + } +} +try { + for (const { scopeDirectory, driverLink } of driverLinks) { + mkdirSync(scopeDirectory, { recursive: true }); + symlinkSync(driverPackage, driverLink, "dir"); + } + const actionSuite = spawnSync( + "pnpm", + [ + "--filter", + "content", + "exec", + "vitest", + "--run", + "actions/migrate-content-database-rows.db.test.ts", + "--config", + "vitest.config.ts", + ], + { + cwd: process.cwd(), + env: { + ...process.env, + CONTENT_MIGRATION_TEST_BACKEND: "pglite", + }, + stdio: "inherit", + }, + ); + if (actionSuite.error) throw actionSuite.error; + if (actionSuite.status !== 0) { + throw new Error( + `PGlite migration action suite failed with status ${actionSuite.status}.`, + ); + } +} finally { + for (const { scopeDirectory, scopeExisted, driverLink } of driverLinks) { + if (existsSync(driverLink)) rmSync(driverLink); + if (!scopeExisted && existsSync(scopeDirectory)) rmdirSync(scopeDirectory); + } +} diff --git a/templates/content/.agents/skills/document-editing/references/databases.md b/templates/content/.agents/skills/document-editing/references/databases.md index 7ab1701548..98dc606f12 100644 --- a/templates/content/.agents/skills/document-editing/references/databases.md +++ b/templates/content/.agents/skills/document-editing/references/databases.md @@ -234,6 +234,19 @@ Use `create-content-database`, `create-inline-content-database`, `duplicate-document-property`, and `delete-document-property`; do not edit property rows or view config via raw SQL when an action can do it. +For a bounded migration that must rewrite every existing row body while adding +new property definitions and values, use `migrate-content-database-rows` rather +than looping the single-row actions. Its `validate` phase is read-only; `apply` +commits the complete plan with an idempotency receipt or writes nothing. Read +the database and every row independently after apply, then call its separate +`verify` phase with the saved post-apply digest. Only a verified receipt may +`finalize` the exact legacy property IDs stored in that receipt. `rollback` is +available before finalize only while the saved post-apply digest still matches, +so it never overwrites a later edit. The bounded migration accepts ordinary +databases without attached Sources; source-backed and system databases retain +their dedicated synchronization actions. If flushing a live row editor changes +its persisted revision, read the rows again and build a fresh plan. + When targeting more than one database row, call `duplicate-database-items` or `remove-database-items` once with a native JSON array of `itemIds` or `documentIds`. Do not loop `duplicate-database-item` or `delete-document` for diff --git a/templates/content/AGENTS.md b/templates/content/AGENTS.md index 639176dc4a..a01c562163 100644 --- a/templates/content/AGENTS.md +++ b/templates/content/AGENTS.md @@ -85,6 +85,7 @@ ladder. | `edit-document` | Find/replace edit — preferred for small changes | | `update-document` | Full rewrite of title, content, or description | | `delete-document` | Move a page and its children to Trash | +| `migrate-content-database-rows` | Validate, atomically apply, verify, roll back, or finalize one bounded whole-database row migration | Every action carries its own schema, and the rest of the app-specific surface (comments, sharing, databases, Notion, local file sources such as diff --git a/templates/content/actions/_content-database-mutation-lock.ts b/templates/content/actions/_content-database-mutation-lock.ts new file mode 100644 index 0000000000..11d857393d --- /dev/null +++ b/templates/content/actions/_content-database-mutation-lock.ts @@ -0,0 +1,41 @@ +import { eq, sql } from "drizzle-orm"; + +import { getDb, schema } from "../server/db/index.js"; +import { withPositionLock } from "./_position-utils.js"; + +export function withContentDatabaseMutationLock( + databaseId: string, + run: () => Promise, +) { + return withPositionLock(`contentDatabaseMutation:${databaseId}`, run); +} + +/** + * Acquire the database row's write lock before changing database membership, + * schema, or every member at once. Callers must hold the returned lock for the + * whole transaction and take it before reading the state they will mutate. + */ +export async function lockContentDatabaseMutation( + tx: ReturnType, + databaseId: string, +) { + const locked = await tx + .update(schema.contentDatabases) + .set({ updatedAt: sql`${schema.contentDatabases.updatedAt}` }) + .where(eq(schema.contentDatabases.id, databaseId)) + .returning({ id: schema.contentDatabases.id }); + if (locked.length !== 1) throw new Error("Database not found."); +} + +export async function touchContentDatabase( + tx: ReturnType, + databaseId: string, + now = new Date().toISOString(), +) { + const touched = await tx + .update(schema.contentDatabases) + .set({ updatedAt: now }) + .where(eq(schema.contentDatabases.id, databaseId)) + .returning({ id: schema.contentDatabases.id }); + if (touched.length !== 1) throw new Error("Database not found."); +} diff --git a/templates/content/actions/_content-database-row-migration.ts b/templates/content/actions/_content-database-row-migration.ts new file mode 100644 index 0000000000..22167c9104 --- /dev/null +++ b/templates/content/actions/_content-database-row-migration.ts @@ -0,0 +1,569 @@ +import { createHash } from "node:crypto"; + +import { and, asc, eq, inArray, isNull } from "drizzle-orm"; +import { z } from "zod"; + +import { schema } from "../server/db/index.js"; +import { + DOCUMENT_PROPERTY_VISIBILITIES, + normalizePropertyValue, + serializePropertyOptions, +} from "../shared/properties.js"; +import { chunks } from "./_batch-utils.js"; + +const propertyType = z.enum(["text", "url", "date", "multi_select"]); +const option = z.object({ + id: z.string().min(1), + name: z.string().min(1), + color: z.enum([ + "gray", + "brown", + "orange", + "yellow", + "green", + "blue", + "purple", + "pink", + "red", + ]), +}); +export const migrationPlanSchema = z.object({ + databaseId: z.string().min(1), + databaseDocumentId: z.string().min(1), + idempotencyKey: z.string().min(1).max(200), + expectedRowCount: z.number().int().min(1).max(100), + propertyDefinitions: z + .array( + z.object({ + id: z.string().min(1), + name: z.string().min(1), + type: propertyType, + visibility: z.enum(DOCUMENT_PROPERTY_VISIBILITIES), + options: z.array(option).max(100).optional(), + }), + ) + .max(100), + rows: z + .array( + z.object({ + itemId: z.string().min(1), + documentId: z.string().min(1), + expectedUpdatedAt: z.string().min(1), + content: z.string(), + propertyValues: z.array( + z.object({ propertyId: z.string().min(1), value: z.unknown() }), + ), + protectedPropertyValues: z + .array( + z.object({ propertyId: z.string().min(1), valueJson: z.string() }), + ) + .max(100), + }), + ) + .max(100), + legacyPropertyIds: z.array(z.string().min(1)).max(100).default([]), +}); +export type MigrationPlan = z.infer; + +export function canonical(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`; + if (value && typeof value === "object") + return `{${Object.entries(value as Record) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([k, v]) => `${JSON.stringify(k)}:${canonical(v)}`) + .join(",")}}`; + return JSON.stringify(value); +} +export const digest = (value: unknown) => + createHash("sha256").update(canonical(value)).digest("hex"); +export const deterministicId = (prefix: string, ...parts: string[]) => + `${prefix}_${digest(parts).slice(0, 28)}`; + +function duplicate(values: string[]) { + return values.length !== new Set(values).size; +} +function valueId(documentId: string, propertyId: string) { + return deterministicId("migration_value", documentId, propertyId); +} + +export async function snapshotMigration(tx: any, databaseId: string) { + const [database] = await tx + .select() + .from(schema.contentDatabases) + .where( + and( + eq(schema.contentDatabases.id, databaseId), + isNull(schema.contentDatabases.deletedAt), + ), + ); + if (!database) throw new Error("Database not found."); + const [databaseDocument] = await tx + .select() + .from(schema.documents) + .where( + and( + eq(schema.documents.id, database.documentId), + eq(schema.documents.ownerEmail, database.ownerEmail), + database.orgId + ? eq(schema.documents.orgId, database.orgId) + : isNull(schema.documents.orgId), + ), + ); + if (!databaseDocument) + throw new Error("Database backing document not found."); + const rows = await tx + .select({ item: schema.contentDatabaseItems, document: schema.documents }) + .from(schema.contentDatabaseItems) + .innerJoin( + schema.documents, + eq(schema.documents.id, schema.contentDatabaseItems.documentId), + ) + .where( + and( + eq(schema.contentDatabaseItems.databaseId, databaseId), + isNull(schema.documents.trashedAt), + ), + ) + .orderBy( + asc(schema.contentDatabaseItems.position), + asc(schema.contentDatabaseItems.id), + ); + const definitions = await tx + .select() + .from(schema.documentPropertyDefinitions) + .where(eq(schema.documentPropertyDefinitions.databaseId, databaseId)) + .orderBy( + asc(schema.documentPropertyDefinitions.position), + asc(schema.documentPropertyDefinitions.id), + ); + const values = rows.length + ? await tx + .select() + .from(schema.documentPropertyValues) + .where( + inArray( + schema.documentPropertyValues.documentId, + rows.map((r: any) => r.document.id), + ), + ) + .orderBy( + asc(schema.documentPropertyValues.documentId), + asc(schema.documentPropertyValues.propertyId), + asc(schema.documentPropertyValues.id), + ) + : []; + const documentIds = [ + database.documentId, + ...rows.map((r: any) => r.document.id), + ]; + const shares = await tx + .select() + .from(schema.documentShares) + .where(inArray(schema.documentShares.resourceId, documentIds)) + .orderBy( + asc(schema.documentShares.resourceId), + asc(schema.documentShares.principalType), + asc(schema.documentShares.principalId), + asc(schema.documentShares.role), + asc(schema.documentShares.id), + ); + const sourceFields = await tx + .select() + .from(schema.contentDatabaseSourceFields) + .where( + inArray( + schema.contentDatabaseSourceFields.propertyId, + definitions.map((definition: any) => definition.id), + ), + ) + .orderBy(asc(schema.contentDatabaseSourceFields.id)); + const sources = await tx + .select({ id: schema.contentDatabaseSources.id }) + .from(schema.contentDatabaseSources) + .where(eq(schema.contentDatabaseSources.databaseId, databaseId)) + .orderBy(asc(schema.contentDatabaseSources.id)); + return { + database, + databaseDocument, + rows, + definitions, + values, + shares, + sourceFields, + sources, + }; +} + +export function validatePlan( + plan: MigrationPlan, + snapshot: Awaited>, + options: { checkTimestamps?: boolean } = {}, +) { + if ( + plan.databaseId !== snapshot.database.id || + plan.databaseDocumentId !== snapshot.database.documentId + ) + throw new Error( + "Plan database identity does not match the active database.", + ); + if (snapshot.database.systemRole || snapshot.sources.length > 0) + throw new Error( + "Migration requires an ordinary database that is not mapped to a source.", + ); + if ( + plan.rows.length !== plan.expectedRowCount || + snapshot.rows.length !== plan.expectedRowCount + ) + throw new Error( + "Migration must include the exact full set of active database rows.", + ); + if ( + duplicate(plan.rows.map((r) => r.itemId)) || + duplicate(plan.rows.map((r) => r.documentId)) + ) + throw new Error("Migration rows must be unique."); + const actual = new Set( + snapshot.rows.map((r: any) => `${r.item.id}:${r.document.id}`), + ); + if (plan.rows.some((r) => !actual.has(`${r.itemId}:${r.documentId}`))) + throw new Error("Migration contains a missing or foreign database row."); + if (duplicate(plan.propertyDefinitions.map((p) => p.id))) + throw new Error("New property definition IDs must be unique."); + const names = plan.propertyDefinitions.map((p) => + p.name.trim().toLocaleLowerCase(), + ); + if (duplicate(names)) + throw new Error("New property definition names must be unique."); + const existingIds = new Set(snapshot.definitions.map((p: any) => p.id)); + const existingNames = new Set( + snapshot.definitions.map((p: any) => p.name.trim().toLocaleLowerCase()), + ); + for (const definition of plan.propertyDefinitions) { + if ( + existingIds.has(definition.id) || + existingNames.has(definition.name.trim().toLocaleLowerCase()) + ) + throw new Error( + "New property definition collides with an existing definition.", + ); + if (definition.type === "multi_select") { + if ( + !definition.options || + duplicate(definition.options.map((o) => o.id)) || + duplicate( + definition.options.map((o) => o.name.trim().toLocaleLowerCase()), + ) + ) + throw new Error( + "Multi-select properties require unique stable option IDs and labels.", + ); + } else if (definition.options?.length) + throw new Error("Only multi-select properties may declare options."); + } + const newDefs = new Map(plan.propertyDefinitions.map((p) => [p.id, p])); + const oldDefs = new Map( + snapshot.definitions.map((p: any) => [p.id, p]), + ); + if (duplicate(plan.legacyPropertyIds)) + throw new Error("Legacy property IDs must be unique."); + for (const propertyId of plan.legacyPropertyIds) { + const definition = oldDefs.get(propertyId); + if (!definition || definition.systemRole || definition.type === "blocks") + throw new Error("Legacy property is missing or unsafe to finalize."); + if ( + snapshot.sourceFields.some( + (field: any) => field.propertyId === propertyId, + ) + ) + throw new Error( + "Legacy properties mapped to a source cannot be migrated.", + ); + if (newDefs.has(propertyId)) + throw new Error("Legacy properties cannot be newly created properties."); + } + for (const row of plan.rows) { + const persisted = snapshot.rows.find( + (candidate: any) => candidate.item.id === row.itemId, + )!; + if ( + options.checkTimestamps !== false && + persisted.document.updatedAt !== row.expectedUpdatedAt + ) + throw new Error(`Stale row ${row.documentId}.`); + if ( + duplicate(row.propertyValues.map((v) => v.propertyId)) || + duplicate(row.protectedPropertyValues.map((v) => v.propertyId)) + ) + throw new Error("Row property values must be unique."); + if (row.propertyValues.length !== newDefs.size) + throw new Error( + "Every row must contain every new property exactly once.", + ); + for (const v of row.propertyValues) { + const definition = newDefs.get(v.propertyId); + if (!definition) + throw new Error("Only newly declared properties may be written."); + normalizeMigrationValue(definition, v.value); + } + for (const protectedValue of row.protectedPropertyValues) { + const definition = oldDefs.get(protectedValue.propertyId); + if ( + !definition || + definition.systemRole || + definition.type === "blocks" || + [ + "formula", + "rollup", + "id", + "created_time", + "created_by", + "last_edited_time", + "last_edited_by", + ].includes(definition.type) + ) + throw new Error("Unsafe protected property target."); + const persistedValue = + snapshot.values.find( + (v: any) => + v.documentId === row.documentId && + v.propertyId === protectedValue.propertyId, + )?.valueJson ?? "null"; + if (persistedValue !== protectedValue.valueJson) + throw new Error( + "Protected property values no longer match persisted values.", + ); + } + } +} + +export function normalizeMigrationValue( + definition: MigrationPlan["propertyDefinitions"][number], + value: unknown, +) { + if (definition.type === "multi_select") { + if (!Array.isArray(value) || value.some((id) => typeof id !== "string")) + throw new Error(`Unknown multi-select option for ${definition.name}.`); + if ( + duplicate(value) || + value.some((id) => !definition.options!.some((o) => o.id === id)) + ) + throw new Error(`Unknown multi-select option for ${definition.name}.`); + return value; + } + const normalized = normalizePropertyValue(definition.type, value); + if (definition.type === "text" && typeof normalized !== "string") + throw new Error(`Text property ${definition.name} must be a string.`); + if (definition.type === "url") { + if (typeof normalized !== "string") + throw new Error(`URL property ${definition.name} must be a string.`); + try { + const url = new URL(normalized); + if (url.protocol !== "http:" && url.protocol !== "https:") + throw new Error(); + } catch { + throw new Error( + `URL property ${definition.name} must be an http/https URL.`, + ); + } + } + if (definition.type === "date" && normalized === null) + throw new Error(`Date property ${definition.name} cannot be null.`); + return normalized; +} + +export function serializeMigrationValue( + definition: MigrationPlan["propertyDefinitions"][number], + value: unknown, +) { + return JSON.stringify(normalizeMigrationValue(definition, value)); +} + +export function snapshotDigest( + snapshot: Awaited>, +) { + const document = (value: any) => ({ + id: value.id, + title: value.title, + content: value.content, + parentId: value.parentId, + ownerEmail: value.ownerEmail, + orgId: value.orgId, + spaceId: value.spaceId, + visibility: value.visibility, + hideFromSearch: value.hideFromSearch, + trashedAt: value.trashedAt, + position: value.position, + }); + return digest({ + database: { + id: snapshot.database.id, + documentId: snapshot.database.documentId, + ownerEmail: snapshot.database.ownerEmail, + orgId: snapshot.database.orgId, + spaceId: snapshot.database.spaceId, + deletedAt: snapshot.database.deletedAt, + }, + databaseDocument: document(snapshot.databaseDocument), + rows: snapshot.rows.map((row: any) => ({ + item: { + id: row.item.id, + databaseId: row.item.databaseId, + documentId: row.item.documentId, + ownerEmail: row.item.ownerEmail, + orgId: row.item.orgId, + position: row.item.position, + }, + document: document(row.document), + })), + definitions: snapshot.definitions.map((definition: any) => ({ + id: definition.id, + databaseId: definition.databaseId, + ownerEmail: definition.ownerEmail, + orgId: definition.orgId, + systemRole: definition.systemRole, + name: definition.name, + type: definition.type, + description: definition.description, + visibility: definition.visibility, + optionsJson: definition.optionsJson, + position: definition.position, + })), + values: snapshot.values.map((value: any) => ({ + id: value.id, + ownerEmail: value.ownerEmail, + documentId: value.documentId, + propertyId: value.propertyId, + valueJson: value.valueJson, + })), + shares: snapshot.shares.map((share: any) => ({ + id: share.id, + resourceId: share.resourceId, + principalType: share.principalType, + principalId: share.principalId, + role: share.role, + })), + sourceFields: snapshot.sourceFields.map((field: any) => ({ + id: field.id, + ownerEmail: field.ownerEmail, + sourceId: field.sourceId, + propertyId: field.propertyId, + localFieldKey: field.localFieldKey, + sourceFieldKey: field.sourceFieldKey, + sourceFieldLabel: field.sourceFieldLabel, + sourceFieldType: field.sourceFieldType, + mappingType: field.mappingType, + writeOwner: field.writeOwner, + readOnly: field.readOnly, + provenance: field.provenance, + freshness: field.freshness, + })), + sources: snapshot.sources, + }); +} + +export async function applyMigration( + tx: any, + plan: MigrationPlan, + snapshot: Awaited>, + receiptId: string, + now: string, +) { + const versions: Array<{ + documentId: string; + versionId: string; + appliedUpdatedAt: string; + }> = []; + const versionRows: Array = []; + for (const row of plan.rows) { + const persisted = snapshot.rows.find( + (candidate: any) => candidate.item.id === row.itemId, + )!; + const versionId = deterministicId( + "migration_version", + receiptId, + row.documentId, + ); + versions.push({ + documentId: row.documentId, + versionId, + appliedUpdatedAt: now, + }); + versionRows.push({ + id: versionId, + ownerEmail: persisted.document.ownerEmail, + documentId: row.documentId, + title: persisted.document.title, + content: persisted.document.content, + createdAt: now, + }); + } + for (const batch of chunks(versionRows, 100)) + await tx.insert(schema.documentVersions).values(batch); + for (const row of plan.rows) { + const updated = await tx + .update(schema.documents) + .set({ content: row.content, updatedAt: now }) + .where( + and( + eq(schema.documents.id, row.documentId), + eq(schema.documents.updatedAt, row.expectedUpdatedAt), + ), + ) + .returning({ id: schema.documents.id }); + if (updated.length !== 1) throw new Error(`Stale row ${row.documentId}.`); + } + const definitionRows = plan.propertyDefinitions.map((definition, index) => ({ + id: definition.id, + ownerEmail: snapshot.database.ownerEmail, + orgId: snapshot.database.orgId, + databaseId: plan.databaseId, + name: definition.name.trim(), + type: definition.type, + visibility: definition.visibility, + optionsJson: serializePropertyOptions( + definition.type === "multi_select" ? { options: definition.options } : {}, + ), + position: + Math.max(-1, ...snapshot.definitions.map((item: any) => item.position)) + + 1 + + index, + createdAt: now, + updatedAt: now, + })); + for (const batch of chunks(definitionRows, 100)) + await tx.insert(schema.documentPropertyDefinitions).values(batch); + const definitionById = new Map( + plan.propertyDefinitions.map((definition) => [definition.id, definition]), + ); + const valueRows: Array = + []; + for (const row of plan.rows) { + const persisted = snapshot.rows.find( + (candidate: any) => candidate.item.id === row.itemId, + )!; + for (const entry of row.propertyValues) + valueRows.push({ + id: valueId(row.documentId, entry.propertyId), + ownerEmail: persisted.document.ownerEmail, + documentId: row.documentId, + propertyId: entry.propertyId, + valueJson: serializeMigrationValue( + definitionById.get(entry.propertyId)!, + entry.value, + ), + createdAt: now, + updatedAt: now, + }); + } + for (const batch of chunks(valueRows, 100)) + await tx.insert(schema.documentPropertyValues).values(batch); + if (plan.rows.length || plan.propertyDefinitions.length) + await tx + .update(schema.contentDatabases) + .set({ updatedAt: now }) + .where(eq(schema.contentDatabases.id, plan.databaseId)); + return { + versions, + createdPropertyIds: plan.propertyDefinitions.map((p) => p.id), + }; +} diff --git a/templates/content/actions/_database-source-utils.ts b/templates/content/actions/_database-source-utils.ts index 683927ee72..f4f537b91b 100644 --- a/templates/content/actions/_database-source-utils.ts +++ b/templates/content/actions/_database-source-utils.ts @@ -86,6 +86,7 @@ import { type ExistingBuilderSourceRowIdentity, } from "./_builder-cms-source-adapter.js"; import { mergeBuilderCmsWriteSettingsIntoJson } from "./_builder-cms-write-settings.js"; +import { lockContentDatabaseMutation } from "./_content-database-mutation-lock.js"; import { ensureDocumentsFilesMembership } from "./_content-files.js"; import { organizationContentSpaceId, @@ -7018,26 +7019,42 @@ export async function replaceSourceMetadata(args: { }) .where(eq(schema.contentDatabaseSources.id, args.source.id)); } else { - await db.insert(schema.contentDatabaseSources).values({ - id: sourceId, - ownerEmail: args.database.ownerEmail, - orgId: args.database.orgId, - databaseId: args.database.id, - sourceType: args.sourceType, - sourceName: args.sourceName, - sourceTable: args.sourceTable, - syncState: "linked", - freshness: "fresh", - capabilitiesJson: sourceCapabilitiesForType(args.sourceType), - metadataJson: serializeSourceMetadataRecord({ + await db.transaction(async (tx) => { + await lockContentDatabaseMutation( + tx as unknown as ReturnType, + args.database.id, + ); + const [existing] = await tx + .select({ id: schema.contentDatabaseSources.id }) + .from(schema.contentDatabaseSources) + .where(eq(schema.contentDatabaseSources.databaseId, args.database.id)) + .limit(1); + if (existing) { + throw new Error( + "Database source state changed before attachment completed.", + ); + } + await tx.insert(schema.contentDatabaseSources).values({ + id: sourceId, + ownerEmail: args.database.ownerEmail, + orgId: args.database.orgId, + databaseId: args.database.id, sourceType: args.sourceType, + sourceName: args.sourceName, sourceTable: args.sourceTable, - }), - lastRefreshedAt: args.now, - lastSourceUpdatedAt: args.now, - lastError: null, - createdAt: args.now, - updatedAt: args.now, + syncState: "linked", + freshness: "fresh", + capabilitiesJson: sourceCapabilitiesForType(args.sourceType), + metadataJson: serializeSourceMetadataRecord({ + sourceType: args.sourceType, + sourceTable: args.sourceTable, + }), + lastRefreshedAt: args.now, + lastSourceUpdatedAt: args.now, + lastError: null, + createdAt: args.now, + updatedAt: args.now, + }); }); } @@ -7050,6 +7067,7 @@ export async function replaceSourceMetadata(args: { */ export async function insertSecondarySource(args: { database: ContentDatabaseRow; + expectedPrimarySourceId: string; sourceType: ContentDatabaseSourceType; sourceName: string; sourceTable: string; @@ -7057,26 +7075,54 @@ export async function insertSecondarySource(args: { }): Promise { const db = getDb(); const sourceId = crypto.randomUUID(); - await db.insert(schema.contentDatabaseSources).values({ - id: sourceId, - ownerEmail: args.database.ownerEmail, - orgId: args.database.orgId, - databaseId: args.database.id, - sourceType: args.sourceType, - sourceName: args.sourceName, - sourceTable: args.sourceTable, - syncState: "linked", - freshness: "fresh", - capabilitiesJson: sourceCapabilitiesForType(args.sourceType), - metadataJson: serializeSourceMetadataRecord({ + await db.transaction(async (tx) => { + await lockContentDatabaseMutation( + tx as unknown as ReturnType, + args.database.id, + ); + const sources = await tx + .select({ + id: schema.contentDatabaseSources.id, + sourceType: schema.contentDatabaseSources.sourceType, + sourceTable: schema.contentDatabaseSources.sourceTable, + }) + .from(schema.contentDatabaseSources) + .where(eq(schema.contentDatabaseSources.databaseId, args.database.id)); + if (!sources.some((source) => source.id === args.expectedPrimarySourceId)) { + throw new Error( + "Database source state changed before attachment completed.", + ); + } + if ( + sources.some( + (source) => + source.sourceType === args.sourceType && + source.sourceTable === args.sourceTable, + ) + ) { + throw new Error(`"${args.sourceTable}" is already attached as a source.`); + } + await tx.insert(schema.contentDatabaseSources).values({ + id: sourceId, + ownerEmail: args.database.ownerEmail, + orgId: args.database.orgId, + databaseId: args.database.id, sourceType: args.sourceType, + sourceName: args.sourceName, sourceTable: args.sourceTable, - }), - lastRefreshedAt: args.now, - lastSourceUpdatedAt: args.now, - lastError: null, - createdAt: args.now, - updatedAt: args.now, + syncState: "linked", + freshness: "fresh", + capabilitiesJson: sourceCapabilitiesForType(args.sourceType), + metadataJson: serializeSourceMetadataRecord({ + sourceType: args.sourceType, + sourceTable: args.sourceTable, + }), + lastRefreshedAt: args.now, + lastSourceUpdatedAt: args.now, + lastError: null, + createdAt: args.now, + updatedAt: args.now, + }); }); return sourceId; } diff --git a/templates/content/actions/_database-utils.ts b/templates/content/actions/_database-utils.ts index d2246c8806..904b66fce7 100644 --- a/templates/content/actions/_database-utils.ts +++ b/templates/content/actions/_database-utils.ts @@ -1448,6 +1448,11 @@ export async function deleteDatabaseDataForDocument( await db .delete(schema.contentDatabaseSources) .where(eq(schema.contentDatabaseSources.databaseId, database.id)); + await db + .delete(schema.contentDatabaseMigrationReceipts) + .where( + eq(schema.contentDatabaseMigrationReceipts.databaseId, database.id), + ); await db .delete(schema.documentPropertyDefinitions) .where(eq(schema.documentPropertyDefinitions.databaseId, database.id)); diff --git a/templates/content/actions/_delete-content-space.ts b/templates/content/actions/_delete-content-space.ts index 3b6a7e9156..3b4d0ff9a4 100644 --- a/templates/content/actions/_delete-content-space.ts +++ b/templates/content/actions/_delete-content-space.ts @@ -2,11 +2,15 @@ import { and, eq } from "drizzle-orm"; import { getDb, schema } from "../server/db/index.js"; import { resolveContentSpaceAccess } from "./_content-space-access.js"; -import { deleteDocumentRecursive } from "./delete-document.js"; +import { + deleteDocumentRootsRecursive, + PermanentDeleteScopeChangedError, +} from "./delete-document.js"; type Db = ReturnType; +const MAX_DELETE_SCOPE_ATTEMPTS = 3; -export async function deleteUserContentSpace(db: Db, spaceId: string) { +async function deleteUserContentSpaceOnce(db: Db, spaceId: string) { return db.transaction(async (tx) => { const scopedDb = tx as unknown as Db; const access = await resolveContentSpaceAccess(spaceId, "editor", { @@ -45,14 +49,9 @@ export async function deleteUserContentSpace(db: Db, spaceId: string) { await scopedDb .delete(schema.contentSpaceCatalogItems) .where(eq(schema.contentSpaceCatalogItems.id, mapping.id)); - const deletedCatalogDocuments = await deleteDocumentRecursive( - scopedDb, - mapping.documentId, - access.space.ownerEmail, - ); - const deletedWorkspaceDocuments = await deleteDocumentRecursive( + const deletedDocuments = await deleteDocumentRootsRecursive( scopedDb, - filesDatabase.documentId, + [mapping.documentId, filesDatabase.documentId], access.space.ownerEmail, ); @@ -77,8 +76,23 @@ export async function deleteUserContentSpace(db: Db, spaceId: string) { return { spaceId, - deletedDocuments: - deletedCatalogDocuments.length + deletedWorkspaceDocuments.length, + deletedDocuments: deletedDocuments.length, }; }); } + +export async function deleteUserContentSpace(db: Db, spaceId: string) { + for (let attempt = 1; attempt <= MAX_DELETE_SCOPE_ATTEMPTS; attempt += 1) { + try { + return await deleteUserContentSpaceOnce(db, spaceId); + } catch (error) { + if ( + !(error instanceof PermanentDeleteScopeChangedError) || + attempt === MAX_DELETE_SCOPE_ATTEMPTS + ) { + throw error; + } + } + } + throw new Error("Workspace deletion did not complete."); +} diff --git a/templates/content/actions/add-database-item.ts b/templates/content/actions/add-database-item.ts index 0cc29cb55e..363d6989b1 100644 --- a/templates/content/actions/add-database-item.ts +++ b/templates/content/actions/add-database-item.ts @@ -10,6 +10,10 @@ import { isComputedPropertyType, type DocumentPropertyType, } from "../shared/properties.js"; +import { + lockContentDatabaseMutation, + touchContentDatabase, +} from "./_content-database-mutation-lock.js"; import { ensureDocumentFilesMembership } from "./_content-files.js"; import { getContentDatabaseResponse } from "./_database-utils.js"; import { @@ -97,54 +101,68 @@ export default defineAction({ .where(eq(schema.documentShares.resourceId, database.documentId)); const initialValues = Object.entries(propertyValues ?? {}); - const propertyValueRows: Array< - typeof schema.documentPropertyValues.$inferInsert - > = []; - if (initialValues.length > 0) { - const requestedPropertyIds = initialValues.map( - ([propertyId]) => propertyId, - ); - const definitions = await db - .select() - .from(schema.documentPropertyDefinitions) - .where( - and( - eq( - schema.documentPropertyDefinitions.ownerEmail, - database.ownerEmail, - ), - eq(schema.documentPropertyDefinitions.databaseId, databaseId), - inArray( - schema.documentPropertyDefinitions.id, - requestedPropertyIds, - ), - ), - ); - const definitionById = new Map( - definitions.map((definition) => [definition.id, definition]), - ); - - for (const [propertyId, value] of initialValues) { - const definition = definitionById.get(propertyId); - const type = definition?.type as DocumentPropertyType | undefined; - if (!definition || !type || isComputedPropertyType(type)) continue; - propertyValueRows.push({ - id: nanoid(), - ownerEmail: database.ownerEmail, - documentId, - propertyId, - valueJson: normalizedValueJson(type, value), - createdAt: now, - updatedAt: now, - }); - } - } await withPositionLock( documentsPositionScope(database.ownerEmail, database.documentId), () => withPositionLock(databaseItemsPositionScope(databaseId), async () => { await db.transaction(async (tx) => { + await lockContentDatabaseMutation( + tx as unknown as ReturnType, + databaseId, + ); + await touchContentDatabase( + tx as unknown as ReturnType, + databaseId, + now, + ); + const propertyValueRows: Array< + typeof schema.documentPropertyValues.$inferInsert + > = []; + if (initialValues.length > 0) { + const requestedPropertyIds = initialValues.map( + ([propertyId]) => propertyId, + ); + const definitions = await tx + .select() + .from(schema.documentPropertyDefinitions) + .where( + and( + eq( + schema.documentPropertyDefinitions.ownerEmail, + database.ownerEmail, + ), + eq( + schema.documentPropertyDefinitions.databaseId, + databaseId, + ), + inArray( + schema.documentPropertyDefinitions.id, + requestedPropertyIds, + ), + ), + ); + const definitionById = new Map( + definitions.map((definition) => [definition.id, definition]), + ); + for (const [propertyId, value] of initialValues) { + const definition = definitionById.get(propertyId); + const type = definition?.type as + | DocumentPropertyType + | undefined; + if (!definition || !type || isComputedPropertyType(type)) + continue; + propertyValueRows.push({ + id: nanoid(), + ownerEmail: database.ownerEmail, + documentId, + propertyId, + valueJson: normalizedValueJson(type, value), + createdAt: now, + updatedAt: now, + }); + } + } const [maxDocPos] = await tx .select({ max: sql`COALESCE(MAX(position), -1)` }) .from(schema.documents) diff --git a/templates/content/actions/attach-content-database-source.ts b/templates/content/actions/attach-content-database-source.ts index 843b819583..4e0374a572 100644 --- a/templates/content/actions/attach-content-database-source.ts +++ b/templates/content/actions/attach-content-database-source.ts @@ -398,6 +398,7 @@ export default defineAction({ const secondaryId = await insertSecondarySource({ database, + expectedPrimarySourceId: existingSource.id, sourceType, sourceName, sourceTable, @@ -491,6 +492,7 @@ export default defineAction({ additionalRead.state === "live" ? additionalRead.entries : []; const additionalSourceId = await insertSecondarySource({ database, + expectedPrimarySourceId: existingSource.id, sourceType, sourceName, sourceTable, diff --git a/templates/content/actions/configure-document-property.ts b/templates/content/actions/configure-document-property.ts index 1e88053c87..3db35a7c95 100644 --- a/templates/content/actions/configure-document-property.ts +++ b/templates/content/actions/configure-document-property.ts @@ -1,7 +1,7 @@ import { defineAction } from "@agent-native/core"; import { writeAppState } from "@agent-native/core/application-state"; import { assertAccess } from "@agent-native/core/sharing"; -import { and, eq, isNull, sql } from "drizzle-orm"; +import { and, eq, sql } from "drizzle-orm"; import { z } from "zod"; import { getDb, schema } from "../server/db/index.js"; @@ -16,6 +16,8 @@ import { normalizePropertyVisibility, type DocumentPropertyType, } from "../shared/properties.js"; +import { lockContentDatabaseMutation } from "./_content-database-mutation-lock.js"; +import { lockDatabaseMemberships } from "./_database-membership-lock.js"; import { propertyDefinitionsPositionScope, withPositionLock, @@ -101,7 +103,7 @@ export default defineAction({ const now = new Date().toISOString(); const name = args.name.trim(); const type = args.type as DocumentPropertyType; - let optionsJson = optionsForNewProperty(type, args.options as any); + const optionsJson = optionsForNewProperty(type, args.options as any); const database = await resolvePropertyDatabaseForDocument( document, args.databaseId, @@ -114,13 +116,12 @@ export default defineAction({ } if (args.id) { - const propertyId = args.id; const [existing] = await db .select() .from(schema.documentPropertyDefinitions) .where( and( - eq(schema.documentPropertyDefinitions.id, propertyId), + eq(schema.documentPropertyDefinitions.id, args.id), eq( schema.documentPropertyDefinitions.ownerEmail, document.ownerEmail, @@ -128,95 +129,79 @@ export default defineAction({ eq(schema.documentPropertyDefinitions.databaseId, database.id), ), ); - if (!existing) throw new Error(`Property "${propertyId}" not found`); - if (existing.systemRole) { - throw new Error("System properties cannot be changed."); - } - if ( - isComputedPropertyType(existing.type as DocumentPropertyType) && - existing.type !== type - ) { - throw new Error("Computed property types cannot be changed."); - } - - const existingOptions = parsePropertyOptions(existing.optionsJson); - const existingIsPrimaryBlocks = - isBlocksPropertyType(existing.type as DocumentPropertyType) && - isPrimaryBlocksField(existingOptions); - - // The primary "Content" Blocks field backs the document body — it can be - // renamed/hidden but not retyped (delete it from the database view to - // remove the body). Block the type switch defensively. - if (existingIsPrimaryBlocks && existing.type !== type) { - throw new Error( - "The primary Content (Blocks) field cannot change type. Delete it from the database view to remove the body.", - ); - } - - // Preserve the primary flag when re-saving the primary Blocks field (a - // rename or visibility change must NOT demote it to a normal Blocks field). - if (existingIsPrimaryBlocks && isBlocksPropertyType(type)) { - optionsJson = serializePropertyOptions({ blocks: { primary: true } }); - } - + if (!existing) throw new Error(`Property "${args.id}" not found`); await db.transaction(async (tx) => { - const [lockedDatabase] = await tx - .update(schema.contentDatabases) - .set({ updatedAt: sql`${schema.contentDatabases.updatedAt}` }) - .where( - and( - eq(schema.contentDatabases.id, database.id), - eq(schema.contentDatabases.documentId, database.documentId), - eq(schema.contentDatabases.ownerEmail, document.ownerEmail), - isNull(schema.contentDatabases.deletedAt), - ), - ) - .returning({ id: schema.contentDatabases.id }); - if (!lockedDatabase) throw new Error("Database is no longer active."); - const [lockedExisting] = await tx - .update(schema.documentPropertyDefinitions) - .set({ - updatedAt: sql`${schema.documentPropertyDefinitions.updatedAt}`, - }) + await lockContentDatabaseMutation( + tx as unknown as ReturnType, + database.id, + ); + let [lockedDefinition] = await tx + .select() + .from(schema.documentPropertyDefinitions) .where( and( - eq(schema.documentPropertyDefinitions.id, propertyId), + eq(schema.documentPropertyDefinitions.id, args.id!), eq( schema.documentPropertyDefinitions.ownerEmail, document.ownerEmail, ), eq(schema.documentPropertyDefinitions.databaseId, database.id), ), - ) - .returning(); - if (!lockedExisting) - throw new Error(`Property "${propertyId}" not found`); - if (lockedExisting.systemRole) + ); + if (!lockedDefinition) + throw new Error(`Property "${args.id}" not found`); + if (lockedDefinition.systemRole) { throw new Error("System properties cannot be changed."); + } if ( - isComputedPropertyType(lockedExisting.type as DocumentPropertyType) && - lockedExisting.type !== type + isComputedPropertyType( + lockedDefinition.type as DocumentPropertyType, + ) && + lockedDefinition.type !== type ) { throw new Error("Computed property types cannot be changed."); } + + const lockedOptions = parsePropertyOptions( + lockedDefinition.optionsJson, + ); const lockedIsPrimaryBlocks = - isBlocksPropertyType(lockedExisting.type as DocumentPropertyType) && - isPrimaryBlocksField( - parsePropertyOptions(lockedExisting.optionsJson), - ); - if (lockedIsPrimaryBlocks && lockedExisting.type !== type) { + isBlocksPropertyType(lockedDefinition.type as DocumentPropertyType) && + isPrimaryBlocksField(lockedOptions); + if (lockedIsPrimaryBlocks && lockedDefinition.type !== type) { throw new Error( "The primary Content (Blocks) field cannot change type. Delete it from the database view to remove the body.", ); } - - if (lockedExisting.type !== type) { + if (lockedDefinition.type !== type) { + const memberships = await tx + .select({ id: schema.contentDatabaseItems.id }) + .from(schema.contentDatabaseItems) + .where(eq(schema.contentDatabaseItems.databaseId, database.id)); + await lockDatabaseMemberships( + tx, + memberships.map((membership) => membership.id), + ); + [lockedDefinition] = await tx + .select() + .from(schema.documentPropertyDefinitions) + .where( + and( + eq(schema.documentPropertyDefinitions.id, args.id!), + eq( + schema.documentPropertyDefinitions.ownerEmail, + document.ownerEmail, + ), + eq(schema.documentPropertyDefinitions.databaseId, database.id), + ), + ); + if (!lockedDefinition) { + throw new Error(`Property "${args.id}" not found`); + } const [mappedSourceField] = await tx .select({ id: schema.contentDatabaseSourceFields.id }) .from(schema.contentDatabaseSourceFields) - .where( - eq(schema.contentDatabaseSourceFields.propertyId, propertyId), - ) + .where(eq(schema.contentDatabaseSourceFields.propertyId, args.id!)) .limit(1); if (mappedSourceField) { throw new Error( @@ -227,7 +212,7 @@ export default defineAction({ .delete(schema.documentPropertyValues) .where( and( - eq(schema.documentPropertyValues.propertyId, propertyId), + eq(schema.documentPropertyValues.propertyId, args.id!), eq( schema.documentPropertyValues.ownerEmail, document.ownerEmail, @@ -239,18 +224,19 @@ export default defineAction({ .where( and( eq(schema.contentDatabaseItemKeyClaims.databaseId, database.id), - eq(schema.contentDatabaseItemKeyClaims.propertyId, propertyId), + eq(schema.contentDatabaseItemKeyClaims.propertyId, args.id!), ), ); - // Switching a Blocks field to another type drops its independent content. if ( - isBlocksPropertyType(lockedExisting.type as DocumentPropertyType) && + isBlocksPropertyType( + lockedDefinition.type as DocumentPropertyType, + ) && !isBlocksPropertyType(type) ) { await tx .delete(schema.documentBlockFieldContents) .where( - eq(schema.documentBlockFieldContents.propertyId, propertyId), + eq(schema.documentBlockFieldContents.propertyId, args.id!), ); } } @@ -265,7 +251,7 @@ export default defineAction({ type, visibility: args.visibility === undefined - ? normalizePropertyVisibility(lockedExisting.visibility) + ? normalizePropertyVisibility(lockedDefinition.visibility) : normalizePropertyVisibility(args.visibility), optionsJson: lockedIsPrimaryBlocks && isBlocksPropertyType(type) @@ -273,40 +259,49 @@ export default defineAction({ : optionsJson, updatedAt: now, }) - .where(eq(schema.documentPropertyDefinitions.id, propertyId)); + .where(eq(schema.documentPropertyDefinitions.id, args.id!)); }); } else { await withPositionLock( propertyDefinitionsPositionScope(database.id), async () => { - const [maxPos] = await db - .select({ - max: sql`COALESCE(MAX(position), -1)`, - }) - .from(schema.documentPropertyDefinitions) - .where( - and( - eq( - schema.documentPropertyDefinitions.ownerEmail, - document.ownerEmail, - ), - eq(schema.documentPropertyDefinitions.databaseId, database.id), - ), + await db.transaction(async (tx) => { + await lockContentDatabaseMutation( + tx as unknown as ReturnType, + database.id, ); + const [maxPos] = await tx + .select({ + max: sql`COALESCE(MAX(position), -1)`, + }) + .from(schema.documentPropertyDefinitions) + .where( + and( + eq( + schema.documentPropertyDefinitions.ownerEmail, + document.ownerEmail, + ), + eq( + schema.documentPropertyDefinitions.databaseId, + database.id, + ), + ), + ); - await db.insert(schema.documentPropertyDefinitions).values({ - id: nanoid(), - ownerEmail: document.ownerEmail, - orgId: document.orgId ?? null, - databaseId: database.id, - name, - description: args.description?.trim() ?? "", - type, - visibility: normalizePropertyVisibility(args.visibility), - optionsJson, - position: (maxPos?.max ?? -1) + 1, - createdAt: now, - updatedAt: now, + await tx.insert(schema.documentPropertyDefinitions).values({ + id: nanoid(), + ownerEmail: document.ownerEmail, + orgId: document.orgId ?? null, + databaseId: database.id, + name, + description: args.description?.trim() ?? "", + type, + visibility: normalizePropertyVisibility(args.visibility), + optionsJson, + position: (maxPos?.max ?? -1) + 1, + createdAt: now, + updatedAt: now, + }); }); }, ); diff --git a/templates/content/actions/content-database-lifecycle.db.test.ts b/templates/content/actions/content-database-lifecycle.db.test.ts index 38941f1900..8eccd6044f 100644 --- a/templates/content/actions/content-database-lifecycle.db.test.ts +++ b/templates/content/actions/content-database-lifecycle.db.test.ts @@ -922,6 +922,7 @@ describe("document trash lifecycle", () => { backingParentId: rootId, deletedAt: databaseDeletedAt, }); + const before = await databaseRow(databaseId); await runWithRequestContext({ userEmail: OWNER }, () => deleteDocumentAction.run({ id: rootId }), @@ -932,6 +933,7 @@ describe("document trash lifecycle", () => { expect(await databaseRow(databaseId)).toMatchObject({ deletedAt: databaseDeletedAt, + updatedAt: before?.updatedAt, }); expect(await documentRow(databaseDocumentId)).toMatchObject({ trashedAt: null, @@ -956,6 +958,72 @@ describe("document trash lifecycle", () => { expect(await documentRow(documentId)).toBeUndefined(); }); + it("removes migration receipts with a permanently deleted database", async () => { + const { databaseId, databaseDocumentId } = await createDatabase({}); + const retainedDatabase = await createDatabase({}); + const receiptId = nextId("migration_receipt"); + const retainedReceiptId = nextId("migration_receipt"); + const stamp = new Date().toISOString(); + await getDb() + .insert(schema.contentDatabaseMigrationReceipts) + .values([ + { + id: receiptId, + ownerEmail: OWNER, + databaseId, + databaseDocumentId, + idempotencyKey: nextId("migration_key"), + planHash: "synthetic-plan-hash", + state: "verified", + preDigest: "synthetic-pre-digest", + postDigest: "synthetic-post-digest", + rollbackJson: JSON.stringify({ content: "synthetic rollback body" }), + resultJson: JSON.stringify({ content: "synthetic migrated body" }), + createdAt: stamp, + updatedAt: stamp, + }, + { + id: retainedReceiptId, + ownerEmail: OWNER, + databaseId: retainedDatabase.databaseId, + databaseDocumentId: retainedDatabase.databaseDocumentId, + idempotencyKey: nextId("migration_key"), + planHash: "retained-plan-hash", + state: "verified", + preDigest: "retained-pre-digest", + postDigest: "retained-post-digest", + rollbackJson: "{}", + resultJson: "{}", + createdAt: stamp, + updatedAt: stamp, + }, + ]); + + await runWithRequestContext({ userEmail: OWNER }, () => + deleteDocumentAction.run({ id: databaseDocumentId }), + ); + await runWithRequestContext({ userEmail: OWNER }, () => + permanentlyDeleteDocumentAction.run({ id: databaseDocumentId }), + ); + + expect( + await getDb() + .select() + .from(schema.contentDatabaseMigrationReceipts) + .where(eq(schema.contentDatabaseMigrationReceipts.id, receiptId)), + ).toHaveLength(0); + expect(await databaseRow(databaseId)).toBeUndefined(); + expect( + await getDb() + .select() + .from(schema.contentDatabaseMigrationReceipts) + .where( + eq(schema.contentDatabaseMigrationReceipts.id, retainedReceiptId), + ), + ).toHaveLength(1); + expect(await databaseRow(retainedDatabase.databaseId)).toBeDefined(); + }); + it("permanently deletes only a selected Trash root", async () => { const rootId = await createDocument({ title: "Trash root" }); const childId = await createDocument({ diff --git a/templates/content/actions/delete-content-database.ts b/templates/content/actions/delete-content-database.ts index eb5e171a0e..5824b6047d 100644 --- a/templates/content/actions/delete-content-database.ts +++ b/templates/content/actions/delete-content-database.ts @@ -5,7 +5,10 @@ import { z } from "zod"; import { getDb } from "../server/db/index.js"; import { assertContentDatabaseLifecycleAccess } from "./_content-database-lifecycle.js"; -import { trashDocumentSubtree } from "./delete-document.js"; +import { + lockDatabasesForTrash, + trashDocumentSubtree, +} from "./delete-document.js"; export default defineAction({ description: @@ -21,12 +24,21 @@ export default defineAction({ await assertAccess("document", database.documentId, "admin"); const db = getDb(); const deletedAt = database.deletedAt ?? new Date().toISOString(); - await trashDocumentSubtree( - db, - database.documentId, - database.ownerEmail, - deletedAt, - ); + await db.transaction(async (tx) => { + const transactionDb = tx as unknown as ReturnType; + const lockedDatabaseIds = await lockDatabasesForTrash( + transactionDb, + database.documentId, + database.ownerEmail, + ); + return trashDocumentSubtree( + transactionDb, + database.documentId, + database.ownerEmail, + deletedAt, + lockedDatabaseIds, + ); + }); await writeAppState("refresh-signal", { ts: Date.now() }); diff --git a/templates/content/actions/delete-document-property.ts b/templates/content/actions/delete-document-property.ts index 61964b002d..9330e9c225 100644 --- a/templates/content/actions/delete-document-property.ts +++ b/templates/content/actions/delete-document-property.ts @@ -1,7 +1,7 @@ import { defineAction } from "@agent-native/core"; import { writeAppState } from "@agent-native/core/application-state"; import { assertAccess } from "@agent-native/core/sharing"; -import { and, eq, inArray, isNull, sql } from "drizzle-orm"; +import { and, eq, inArray } from "drizzle-orm"; import { z } from "zod"; import { getDb, schema } from "../server/db/index.js"; @@ -11,6 +11,8 @@ import { parsePropertyOptions, type DocumentPropertyType, } from "../shared/properties.js"; +import { lockContentDatabaseMutation } from "./_content-database-mutation-lock.js"; +import { lockDatabaseMemberships } from "./_database-membership-lock.js"; import { listPropertiesForDocument, resolvePropertyDatabaseForDocument, @@ -58,32 +60,22 @@ export default defineAction({ throw new Error("System properties cannot be deleted."); } - const isBlocks = isBlocksPropertyType( - definition.type as DocumentPropertyType, - ); - const isPrimaryBlocks = - isBlocks && - isPrimaryBlocksField(parsePropertyOptions(definition.optionsJson)); - await db.transaction(async (tx) => { - const [lockedDatabase] = await tx - .update(schema.contentDatabases) - .set({ updatedAt: sql`${schema.contentDatabases.updatedAt}` }) - .where( - and( - eq(schema.contentDatabases.id, database.id), - eq(schema.contentDatabases.documentId, database.documentId), - eq(schema.contentDatabases.ownerEmail, document.ownerEmail), - isNull(schema.contentDatabases.deletedAt), - ), - ) - .returning({ id: schema.contentDatabases.id }); - if (!lockedDatabase) throw new Error("Database is no longer active."); + await lockContentDatabaseMutation( + tx as unknown as ReturnType, + database.id, + ); + const memberships = await tx + .select({ id: schema.contentDatabaseItems.id }) + .from(schema.contentDatabaseItems) + .where(eq(schema.contentDatabaseItems.databaseId, database.id)); + await lockDatabaseMemberships( + tx, + memberships.map((membership) => membership.id), + ); const [lockedDefinition] = await tx - .update(schema.documentPropertyDefinitions) - .set({ - updatedAt: sql`${schema.documentPropertyDefinitions.updatedAt}`, - }) + .select() + .from(schema.documentPropertyDefinitions) .where( and( eq(schema.documentPropertyDefinitions.id, propertyId), @@ -93,15 +85,21 @@ export default defineAction({ ), eq(schema.documentPropertyDefinitions.databaseId, database.id), ), - ) - .returning({ - id: schema.documentPropertyDefinitions.id, - systemRole: schema.documentPropertyDefinitions.systemRole, - }); + ); if (!lockedDefinition) throw new Error(`Property "${propertyId}" not found`); - if (lockedDefinition.systemRole) + if (lockedDefinition.systemRole) { throw new Error("System properties cannot be deleted."); + } + + const isBlocks = isBlocksPropertyType( + lockedDefinition.type as DocumentPropertyType, + ); + const isPrimaryBlocks = + isBlocks && + isPrimaryBlocksField( + parsePropertyOptions(lockedDefinition.optionsJson), + ); await tx .delete(schema.documentPropertyValues) @@ -119,19 +117,11 @@ export default defineAction({ .where(eq(schema.documentPropertyDefinitions.id, propertyId)); if (isBlocks) { - // Drop the independent content for this Blocks field across every row. await tx .delete(schema.documentBlockFieldContents) .where(eq(schema.documentBlockFieldContents.propertyId, propertyId)); - // Deleting the primary "Content" field removes the body (documents.content) - // for every object of this type, per the delete warning shown in the UI. if (isPrimaryBlocks) { - // Record that the primary was intentionally removed: clear the single - // source of truth but LEAVE blocks_seeded = 1, so neither the read path - // nor the startup repair ever recreates it. Deleting the only Blocks - // field is an allowed product action that leaves the row metadata-only - // with ZERO Blocks fields. await tx .update(schema.contentDatabases) .set({ @@ -144,20 +134,21 @@ export default defineAction({ .select({ documentId: schema.contentDatabaseItems.documentId }) .from(schema.contentDatabaseItems) .where(eq(schema.contentDatabaseItems.databaseId, database.id)); - const documentIds = items.map((item) => item.documentId); - if (documentIds.length > 0) { + if (items.length > 0) { const now = new Date().toISOString(); await tx .update(schema.documents) .set({ content: "", updatedAt: now }) - .where(inArray(schema.documents.id, documentIds)); + .where( + inArray( + schema.documents.id, + items.map((item) => item.documentId), + ), + ); } } } - // Free any source field that was mapped to this property so it returns to - // the "From source" picker immediately, instead of staying orphaned until - // the next source refresh reconciles it. const mappedFields = await tx .select({ id: schema.contentDatabaseSourceFields.id, @@ -165,19 +156,17 @@ export default defineAction({ }) .from(schema.contentDatabaseSourceFields) .where(eq(schema.contentDatabaseSourceFields.propertyId, propertyId)); - if (mappedFields.length > 0) { - const now = new Date().toISOString(); - for (const mapped of mappedFields) { - await tx - .update(schema.contentDatabaseSourceFields) - .set({ - propertyId: null, - localFieldKey: mapped.sourceFieldKey, - mappingType: "property", - updatedAt: now, - }) - .where(eq(schema.contentDatabaseSourceFields.id, mapped.id)); - } + const now = new Date().toISOString(); + for (const mapped of mappedFields) { + await tx + .update(schema.contentDatabaseSourceFields) + .set({ + propertyId: null, + localFieldKey: mapped.sourceFieldKey, + mappingType: "property", + updatedAt: now, + }) + .where(eq(schema.contentDatabaseSourceFields.id, mapped.id)); } }); diff --git a/templates/content/actions/delete-document.test.ts b/templates/content/actions/delete-document.test.ts index b671f10d6f..4248a4a96d 100644 --- a/templates/content/actions/delete-document.test.ts +++ b/templates/content/actions/delete-document.test.ts @@ -12,6 +12,18 @@ vi.mock("drizzle-orm", async (importOriginal) => { }; }); +const mutationLock = vi.hoisted(() => vi.fn()); +const membershipLock = vi.hoisted(() => vi.fn()); + +vi.mock("./_content-database-mutation-lock.js", () => ({ + lockContentDatabaseMutation: mutationLock, + touchContentDatabase: vi.fn(), +})); + +vi.mock("./_database-membership-lock.js", () => ({ + lockDatabaseMemberships: membershipLock, +})); + // Minimal schema stand-in: each table is identified by name so a fake db can // record which table a delete/select targeted. const { schema } = vi.hoisted(() => ({ @@ -27,6 +39,7 @@ const { schema } = vi.hoisted(() => ({ ownerEmail: "contentDatabases.ownerEmail", }, contentDatabaseItems: { + id: "contentDatabaseItems.id", databaseId: "contentDatabaseItems.databaseId", documentId: "contentDatabaseItems.documentId", ownerEmail: "contentDatabaseItems.ownerEmail", @@ -92,6 +105,9 @@ const { schema } = vi.hoisted(() => ({ ownerEmail: "documentComments.ownerEmail", }, documentShares: { resourceId: "documentShares.resourceId" }, + contentDatabaseMigrationReceipts: { + databaseId: "contentDatabaseMigrationReceipts.databaseId", + }, }, })); @@ -131,6 +147,8 @@ describe("deleteDocumentRecursive", () => { let operationsOutsideTransaction: string[]; beforeEach(() => { + mutationLock.mockReset(); + membershipLock.mockReset(); deleteCalls = []; selectRows = { documents: [], @@ -197,30 +215,9 @@ describe("deleteDocumentRecursive", () => { ownerEmail: "owner-a@example.com", }, ]; - let lockedParentDatabase = false; - db.update = (table: Record) => ({ - set: () => ({ - where: async (cond: unknown) => { - if ( - table === schema.contentDatabases && - matches( - { - id: "parent-database", - ownerEmail: "owner-a@example.com", - }, - cond, - ) - ) { - lockedParentDatabase = true; - } - return []; - }, - }), - }); - await deleteDocumentRecursive(db, "row-document", "owner-a@example.com"); - expect(lockedParentDatabase).toBe(true); + expect(mutationLock).toHaveBeenCalledWith(db, "parent-database"); const parentDatabaseDeletes = deleteCalls.filter( (call) => call.table === "contentDatabases", ); @@ -273,11 +270,13 @@ describe("deleteDocumentRecursive", () => { ]; selectRows.contentDatabaseItems = [ { + id: "membership-1", databaseId: "database-1", documentId: "row-doc-1", ownerEmail: "owner-a@example.com", }, { + id: "membership-2", databaseId: "database-1", documentId: "row-doc-2", ownerEmail: "owner-a@example.com", @@ -307,6 +306,11 @@ describe("deleteDocumentRecursive", () => { expect(membershipDeletes[0].cond).toEqual({ __inArray: [schema.contentDatabaseItems.databaseId, ["database-1"]], }); + expect(mutationLock).toHaveBeenCalledWith(db, "database-1"); + expect(membershipLock).toHaveBeenCalledWith( + db, + expect.arrayContaining(["membership-1", "membership-2"]), + ); }); it("recollects database rows after acquiring the permanent-cleanup lock", async () => { @@ -327,22 +331,16 @@ describe("deleteDocumentRecursive", () => { selectRows.documents = [ { id: "row-doc-1", ownerEmail: "owner-a@example.com" }, ]; - db.update = () => ({ - set: () => ({ - where: async () => { - if (transactionDepth === 0) - operationsOutsideTransaction.push("update:contentDatabases"); - selectRows.contentDatabaseItems.push({ - databaseId: "database-1", - documentId: "late-row-doc", - ownerEmail: "owner-a@example.com", - }); - selectRows.documents.push({ - id: "late-row-doc", - ownerEmail: "owner-a@example.com", - }); - }, - }), + mutationLock.mockImplementationOnce(async () => { + selectRows.contentDatabaseItems.push({ + databaseId: "database-1", + documentId: "late-row-doc", + ownerEmail: "owner-a@example.com", + }); + selectRows.documents.push({ + id: "late-row-doc", + ownerEmail: "owner-a@example.com", + }); }); const deleted = await deleteDocumentRecursive( @@ -367,16 +365,19 @@ describe("deleteDocumentRecursive", () => { ]; selectRows.contentDatabaseItems = [ { + id: "membership-1", databaseId: "database-1", documentId: "row-doc-1", ownerEmail: "owner-a@example.com", }, { + id: "membership-foreign", databaseId: "database-1", documentId: "foreign-row-doc", ownerEmail: "owner-b@example.com", }, { + id: "membership-mismatched", databaseId: "database-1", documentId: "mismatched-row-doc", ownerEmail: "owner-a@example.com", diff --git a/templates/content/actions/delete-document.ts b/templates/content/actions/delete-document.ts index df7e70cd18..855c0eba33 100644 --- a/templates/content/actions/delete-document.ts +++ b/templates/content/actions/delete-document.ts @@ -1,16 +1,38 @@ import { defineAction } from "@agent-native/core"; import { writeAppState } from "@agent-native/core/application-state"; import { assertAccess } from "@agent-native/core/sharing"; -import { and, eq, inArray, isNotNull, isNull, ne, or, sql } from "drizzle-orm"; +import { and, eq, inArray, isNotNull, isNull, ne, or } from "drizzle-orm"; import { z } from "zod"; import { getDb, schema } from "../server/db/index.js"; import { chunks } from "./_batch-utils.js"; +import { + lockContentDatabaseMutation, + touchContentDatabase, +} from "./_content-database-mutation-lock.js"; import { assertNotWorkspaceCatalogDocuments } from "./_content-space-catalog-guards.js"; +import { lockDatabaseMemberships } from "./_database-membership-lock.js"; import { renumberDatabaseRows } from "./_database-row-batch.js"; const DELETE_BATCH_SIZE = 90; +export class PermanentDeleteScopeChangedError extends Error { + constructor() { + super("Document deletion scope changed; retry deletion."); + } +} + +type PermanentDeleteScope = { + documentIds: string[]; + ownedDatabaseIds: string[]; +}; + +function hasSameIds(left: string[], right: string[]) { + if (left.length !== right.length) return false; + const rightIds = new Set(right); + return left.every((id) => rightIds.has(id)); +} + async function selectDocumentChildren( db: ReturnType, parentIds: string[], @@ -97,26 +119,75 @@ async function selectDatabaseItemDocuments( return ownedRows.map((row) => ({ documentId: row.id })); } -async function selectMembershipDatabaseIds( +async function selectDatabaseItemDocumentTrashState( db: ReturnType, - documentIds: string[], + databaseIds: string[], ownerEmail: string, ) { - const rows: Array<{ databaseId: string }> = []; - for (const batch of chunks(documentIds, DELETE_BATCH_SIZE)) { + const itemDocuments = await selectDatabaseItemDocuments( + db, + databaseIds, + ownerEmail, + ); + const rows: Array<{ + id: string; + trashedAt: string | null; + }> = []; + for (const batch of chunks( + itemDocuments.map((item) => item.documentId), + DELETE_BATCH_SIZE, + )) { rows.push( ...(await db - .select({ databaseId: schema.contentDatabaseItems.databaseId }) - .from(schema.contentDatabaseItems) + .select({ + id: schema.documents.id, + trashedAt: schema.documents.trashedAt, + }) + .from(schema.documents) .where( and( - inArray(schema.contentDatabaseItems.documentId, batch), - eq(schema.contentDatabaseItems.ownerEmail, ownerEmail), + inArray(schema.documents.id, batch), + eq(schema.documents.ownerEmail, ownerEmail), ), )), ); } - return [...new Set(rows.map((row) => row.databaseId))]; + return rows; +} + +async function selectMembershipsForDocuments( + db: ReturnType, + documentIds: string[], +) { + const rows: Array<{ id: string; databaseId: string }> = []; + for (const batch of chunks(documentIds, DELETE_BATCH_SIZE)) { + rows.push( + ...(await db + .select({ + id: schema.contentDatabaseItems.id, + databaseId: schema.contentDatabaseItems.databaseId, + }) + .from(schema.contentDatabaseItems) + .where(inArray(schema.contentDatabaseItems.documentId, batch))), + ); + } + return rows; +} + +async function selectMembershipIdsForDatabases( + db: ReturnType, + databaseIds: string[], +) { + const rows: Array<{ id: string }> = []; + for (const batch of chunks(databaseIds, DELETE_BATCH_SIZE)) { + rows.push( + ...(await db + .select({ id: schema.contentDatabaseItems.id }) + .from(schema.contentDatabaseItems) + .where(inArray(schema.contentDatabaseItems.databaseId, batch))), + ); + } + return rows.map((row) => row.id); } async function collectDocumentSubtreeForDelete( @@ -172,84 +243,117 @@ async function collectDocumentSubtreeForDelete( frontier = [...next]; } - const collectedDocumentIds = [...documentIds]; - const collectedOwnedDatabaseIds = [...ownedDatabaseIds]; return { - documentIds: collectedDocumentIds, - ownedDatabaseIds: collectedOwnedDatabaseIds, - lockDatabaseIds: [ - ...new Set([ - ...collectedOwnedDatabaseIds, - ...(await selectMembershipDatabaseIds( - db, - collectedDocumentIds, - ownerEmail, - )), - ]), - ], + documentIds: [...documentIds], + ownedDatabaseIds: [...ownedDatabaseIds], }; } -async function lockDatabasesAndRecollect< - T extends { lockDatabaseIds: string[] }, ->( +async function lockPermanentDeleteScope( db: ReturnType, - ownerEmail: string, - collect: () => Promise, -): Promise { - const lockedDatabaseIds = new Set(); - let collected = await collect(); - while (true) { - const unlockedDatabaseIds = collected.lockDatabaseIds.filter( - (databaseId) => !lockedDatabaseIds.has(databaseId), - ); - if (unlockedDatabaseIds.length === 0) return collected; - for (const batch of chunks(unlockedDatabaseIds, DELETE_BATCH_SIZE)) { - await db - .update(schema.contentDatabases) - .set({ updatedAt: sql`${schema.contentDatabases.updatedAt}` }) - .where( - and( - inArray(schema.contentDatabases.id, batch), - eq(schema.contentDatabases.ownerEmail, ownerEmail), - ), - ); - for (const databaseId of batch) lockedDatabaseIds.add(databaseId); - } - collected = await collect(); + collectScope: () => Promise, +) { + const initialScope = await collectScope(); + const initialMemberships = await selectMembershipsForDocuments( + db, + initialScope.documentIds, + ); + const lockedDatabaseIds = [ + ...new Set([ + ...initialScope.ownedDatabaseIds, + ...initialMemberships.map((membership) => membership.databaseId), + ]), + ].sort(); + for (const databaseId of lockedDatabaseIds) { + await lockContentDatabaseMutation(db, databaseId); + } + + const reloadedScope = await collectScope(); + const reloadedMemberships = await selectMembershipsForDocuments( + db, + reloadedScope.documentIds, + ); + const lockedDatabaseIdSet = new Set(lockedDatabaseIds); + const uncoveredDatabaseId = [ + ...reloadedScope.ownedDatabaseIds, + ...reloadedMemberships.map((membership) => membership.databaseId), + ].find((databaseId) => !lockedDatabaseIdSet.has(databaseId)); + if (uncoveredDatabaseId) { + throw new PermanentDeleteScopeChangedError(); + } + + const membershipIds = await selectMembershipIdsForDatabases( + db, + lockedDatabaseIds, + ); + await lockDatabaseMemberships(db, membershipIds); + + const lockedScope = await collectScope(); + const lockedMemberships = await selectMembershipsForDocuments( + db, + lockedScope.documentIds, + ); + const lockedMembershipIds = await selectMembershipIdsForDatabases( + db, + lockedDatabaseIds, + ); + if ( + !hasSameIds(reloadedScope.documentIds, lockedScope.documentIds) || + !hasSameIds(reloadedScope.ownedDatabaseIds, lockedScope.ownedDatabaseIds) || + !hasSameIds(membershipIds, lockedMembershipIds) || + [ + ...lockedScope.ownedDatabaseIds, + ...lockedMemberships.map((membership) => membership.databaseId), + ].some((databaseId) => !lockedDatabaseIdSet.has(databaseId)) + ) { + throw new PermanentDeleteScopeChangedError(); } + return lockedScope; } -export async function trashDocumentSubtree( +export async function lockDatabasesForTrash( db: ReturnType, id: string, ownerEmail: string, - trashedAt = new Date().toISOString(), -): Promise { - return db.transaction((tx) => - trashDocumentSubtreeInTransaction( - tx as unknown as ReturnType, - id, - ownerEmail, - trashedAt, - ), +) { + const subtree = await collectDocumentSubtreeForDelete(db, id, ownerEmail); + const memberships = await selectMembershipsForDocuments( + db, + subtree.documentIds, ); + const databaseIds = [ + ...new Set([ + ...memberships.map((membership) => membership.databaseId), + ...subtree.ownedDatabaseIds, + ]), + ].sort(); + for (const databaseId of databaseIds) { + await lockContentDatabaseMutation(db, databaseId); + } + return new Set(databaseIds); } -async function trashDocumentSubtreeInTransaction( +export async function trashDocumentSubtree( db: ReturnType, id: string, ownerEmail: string, - trashedAt: string, + trashedAt = new Date().toISOString(), + lockedDatabaseIds?: ReadonlySet, ): Promise { - // Stable-key upserts lock their canonical database row before creating or - // updating children. Acquire the same locks, then collect again: an upsert - // that won the lock before this transaction may have added a child after the - // first traversal, while one that arrives later must wait until the database - // has been marked deleted and will fail closed. - const { documentIds } = await lockDatabasesAndRecollect(db, ownerEmail, () => - collectDocumentSubtreeForDelete(db, id, ownerEmail), - ); + const { documentIds, ownedDatabaseIds } = + await collectDocumentSubtreeForDelete(db, id, ownerEmail); + if (lockedDatabaseIds) { + const memberships = await selectMembershipsForDocuments(db, documentIds); + const unlockedDatabaseId = [ + ...new Set([ + ...ownedDatabaseIds, + ...memberships.map((membership) => membership.databaseId), + ]), + ].find((databaseId) => !lockedDatabaseIds.has(databaseId)); + if (unlockedDatabaseId) { + throw new Error("Document subtree changed; retry deletion."); + } + } await assertNotWorkspaceCatalogDocuments(db, documentIds, "deleted"); const independentlyTrashedDatabaseDocumentIds = new Set(); @@ -291,6 +395,39 @@ async function trashDocumentSubtreeInTransaction( ); } + const activeMemberships = await selectMembershipsForDocuments( + db, + activeDocumentIds, + ); + const transitioningOwnedDatabases: Array<{ id: string }> = []; + for (const batch of chunks(activeDocumentIds, DELETE_BATCH_SIZE)) { + transitioningOwnedDatabases.push( + ...(await db + .select({ id: schema.contentDatabases.id }) + .from(schema.contentDatabases) + .where( + and( + inArray(schema.contentDatabases.documentId, batch), + eq(schema.contentDatabases.ownerEmail, ownerEmail), + isNull(schema.contentDatabases.deletedAt), + ), + )), + ); + } + const transitioningOwnedDatabaseIds = new Set( + transitioningOwnedDatabases.map((database) => database.id), + ); + const externalDatabaseIds = [ + ...new Set( + activeMemberships + .map((membership) => membership.databaseId) + .filter((databaseId) => !transitioningOwnedDatabaseIds.has(databaseId)), + ), + ].sort(); + for (const databaseId of externalDatabaseIds) { + await touchContentDatabase(db, databaseId, trashedAt); + } + for (const batch of chunks(activeDocumentIds, DELETE_BATCH_SIZE)) { await db .update(schema.documents) @@ -322,19 +459,55 @@ export async function restoreDocumentSubtree( rootId: string, ownerEmail: string, ): Promise { - const documentIds = ( - await db - .select({ id: schema.documents.id }) - .from(schema.documents) - .where( - and( - eq(schema.documents.trashRootId, rootId), - eq(schema.documents.ownerEmail, ownerEmail), - ), - ) - ).map((document) => document.id); - if (documentIds.length === 0) return []; + const collectRestoreScope = async () => { + const documentIds = ( + await db + .select({ id: schema.documents.id }) + .from(schema.documents) + .where( + and( + eq(schema.documents.trashRootId, rootId), + eq(schema.documents.ownerEmail, ownerEmail), + ), + ) + ).map((document) => document.id); + const memberships = await selectMembershipsForDocuments(db, documentIds); + const ownedDatabaseIds = ( + await selectOwnedDatabaseIds(db, documentIds, ownerEmail) + ).map((database) => database.id); + return { documentIds, memberships, ownedDatabaseIds }; + }; + + const initialScope = await collectRestoreScope(); + if (initialScope.documentIds.length === 0) return []; + const lockedDatabaseIds = [ + ...new Set([ + ...initialScope.ownedDatabaseIds, + ...initialScope.memberships.map((membership) => membership.databaseId), + ]), + ].sort(); + for (const databaseId of lockedDatabaseIds) { + await lockContentDatabaseMutation(db, databaseId); + } + const restoreScope = await collectRestoreScope(); + const lockedDatabaseIdSet = new Set(lockedDatabaseIds); + const unlockedDatabaseId = [ + ...new Set([ + ...restoreScope.ownedDatabaseIds, + ...restoreScope.memberships.map((membership) => membership.databaseId), + ]), + ].find((databaseId) => !lockedDatabaseIdSet.has(databaseId)); + if (unlockedDatabaseId) { + throw new Error("Document restore scope changed; retry restoration."); + } + await lockDatabaseMemberships( + db, + await selectMembershipIdsForDatabases(db, lockedDatabaseIds), + ); + + const documentIds = restoreScope.documentIds; + if (documentIds.length === 0) return []; const now = new Date().toISOString(); for (const batch of chunks(documentIds, DELETE_BATCH_SIZE)) { await db @@ -399,20 +572,49 @@ export async function deleteDocumentRecursive( id: string, ownerEmail: string, ): Promise { - return db.transaction(async (tx) => { - const scopedDb = tx as unknown as ReturnType; - const { documentIds, ownedDatabaseIds } = await lockDatabasesAndRecollect( - scopedDb, - ownerEmail, - () => collectDocumentSubtreeForDelete(scopedDb, id, ownerEmail), - ); - return deleteCollectedDocuments( - scopedDb, - documentIds, - ownedDatabaseIds, + return db.transaction((tx) => + deleteDocumentRootsRecursive( + tx as unknown as ReturnType, + [id], ownerEmail, - ); - }); + ), + ); +} + +export async function deleteDocumentRootsRecursive( + db: ReturnType, + rootIds: string[], + ownerEmail: string, +): Promise { + const collectScope = async () => { + const documentIds = new Set(); + const ownedDatabaseIds = new Set(); + for (const rootId of rootIds) { + const scope = await collectDocumentSubtreeForDelete( + db, + rootId, + ownerEmail, + ); + for (const documentId of scope.documentIds) documentIds.add(documentId); + for (const databaseId of scope.ownedDatabaseIds) { + ownedDatabaseIds.add(databaseId); + } + } + return { + documentIds: [...documentIds], + ownedDatabaseIds: [...ownedDatabaseIds], + }; + }; + const { documentIds, ownedDatabaseIds } = await lockPermanentDeleteScope( + db, + collectScope, + ); + return deleteCollectedDocuments( + db, + documentIds, + ownedDatabaseIds, + ownerEmail, + ); } async function deleteCollectedDocuments( @@ -515,6 +717,7 @@ async function deleteCollectedDocuments( ), ); }); + await deleteWhereIn(documentIds, async (documentIdBatch) => { await db .delete(schema.contentDatabaseItemKeyClaims) @@ -570,6 +773,17 @@ async function deleteCollectedDocuments( await db .delete(schema.contentDatabases) .where(inArray(schema.contentDatabases.id, databaseIdBatch)); + // Receipts deliberately have no database foreign key. Removing them after + // the database row closes the race with a migration that already holds the + // row lock and commits its receipt before this deletion can continue. + await db + .delete(schema.contentDatabaseMigrationReceipts) + .where( + inArray( + schema.contentDatabaseMigrationReceipts.databaseId, + databaseIdBatch, + ), + ); }); await deleteWhereIn(documentIds, async (documentIdBatch) => { @@ -629,74 +843,63 @@ export async function deleteTrashedDocumentSubtree( id: string, ownerEmail: string, ): Promise { - return db.transaction((tx) => - deleteTrashedDocumentSubtreeInTransaction( - tx as unknown as ReturnType, - id, - ownerEmail, - ), - ); -} + const collectScope = async () => { + const [root] = await db + .select({ id: schema.documents.id }) + .from(schema.documents) + .where( + and( + eq(schema.documents.id, id), + eq(schema.documents.ownerEmail, ownerEmail), + eq(schema.documents.trashRootId, id), + isNotNull(schema.documents.trashedAt), + ), + ) + .limit(1); + if (!root) { + throw new Error( + "Document must be in Trash and be a Trash root before permanent deletion", + ); + } -async function deleteTrashedDocumentSubtreeInTransaction( - db: ReturnType, - id: string, - ownerEmail: string, -): Promise { - const [root] = await db - .select({ id: schema.documents.id }) - .from(schema.documents) - .where( - and( - eq(schema.documents.id, id), - eq(schema.documents.ownerEmail, ownerEmail), - eq(schema.documents.trashRootId, id), - isNotNull(schema.documents.trashedAt), - ), - ) - .limit(1); - if (!root) { - throw new Error( - "Document must be in Trash and be a Trash root before permanent deletion", + const documentIds = ( + await db + .select({ id: schema.documents.id }) + .from(schema.documents) + .where( + and( + eq(schema.documents.ownerEmail, ownerEmail), + eq(schema.documents.trashRootId, id), + isNotNull(schema.documents.trashedAt), + ), + ) + ).map((document) => document.id); + const ownedDatabaseIds = await selectOwnedDatabaseIds( + db, + documentIds, + ownerEmail, + ).then((rows) => rows.map((database) => database.id)); + const documentIdSet = new Set(documentIds); + const activeOutsideScope = ( + await selectDatabaseItemDocumentTrashState( + db, + ownedDatabaseIds, + ownerEmail, + ) + ).find( + (document) => !documentIdSet.has(document.id) && !document.trashedAt, ); - } + if (activeOutsideScope) { + throw new Error( + "Database contains an active row outside this Trash item", + ); + } + return { documentIds, ownedDatabaseIds }; + }; - const { documentIds, ownedDatabaseIds } = await lockDatabasesAndRecollect( + const { documentIds, ownedDatabaseIds } = await lockPermanentDeleteScope( db, - ownerEmail, - async () => { - const collectedDocumentIds = ( - await db - .select({ id: schema.documents.id }) - .from(schema.documents) - .where( - and( - eq(schema.documents.ownerEmail, ownerEmail), - eq(schema.documents.trashRootId, id), - isNotNull(schema.documents.trashedAt), - ), - ) - ).map((document) => document.id); - const collectedDatabaseIds = await selectOwnedDatabaseIds( - db, - collectedDocumentIds, - ownerEmail, - ).then((rows) => rows.map((database) => database.id)); - return { - documentIds: collectedDocumentIds, - ownedDatabaseIds: collectedDatabaseIds, - lockDatabaseIds: [ - ...new Set([ - ...collectedDatabaseIds, - ...(await selectMembershipDatabaseIds( - db, - collectedDocumentIds, - ownerEmail, - )), - ]), - ], - }; - }, + collectScope, ); await db @@ -760,9 +963,19 @@ export default defineAction({ if (!membership) { throw new Error("Document is not part of Favorites"); } - await db - .delete(schema.contentDatabaseItems) - .where(eq(schema.contentDatabaseItems.id, membership.id)); + await db.transaction(async (tx) => { + await lockContentDatabaseMutation( + tx as unknown as ReturnType, + contextDatabase.id, + ); + await touchContentDatabase( + tx as unknown as ReturnType, + contextDatabase.id, + ); + await tx + .delete(schema.contentDatabaseItems) + .where(eq(schema.contentDatabaseItems.id, membership.id)); + }); await writeAppState("refresh-signal", { ts: Date.now() }); return { success: true, deleted: 0, removed: 1 }; } @@ -777,11 +990,21 @@ export default defineAction({ if (systemDatabase?.systemRole) { throw new Error("System Content database documents cannot be deleted"); } - const deleted = await trashDocumentSubtree( - db, - id, - existing.ownerEmail as string, - ); + const deleted = await db.transaction(async (tx) => { + const transactionDb = tx as unknown as ReturnType; + const lockedDatabaseIds = await lockDatabasesForTrash( + transactionDb, + id, + existing.ownerEmail as string, + ); + return trashDocumentSubtree( + transactionDb, + id, + existing.ownerEmail as string, + undefined, + lockedDatabaseIds, + ); + }); await writeAppState("refresh-signal", { ts: Date.now() }); diff --git a/templates/content/actions/disconnect-content-database-source.ts b/templates/content/actions/disconnect-content-database-source.ts index 092bdd5abd..f70ed00a95 100644 --- a/templates/content/actions/disconnect-content-database-source.ts +++ b/templates/content/actions/disconnect-content-database-source.ts @@ -8,35 +8,52 @@ import type { ContentDatabaseResponse, DisconnectContentDatabaseSourceRequest, } from "../shared/api.js"; +import { lockContentDatabaseMutation } from "./_content-database-mutation-lock.js"; import { getExistingSource, resolveDatabaseForSourceMutation, } from "./_database-source-utils.js"; import { getContentDatabaseResponse } from "./_database-utils.js"; -async function deleteSourceRecords(sourceId: string) { +async function deleteSourceRecords(databaseId: string, sourceId: string) { const db = getDb(); - await db - .delete(schema.contentDatabaseBodyHydrationQueue) - .where(eq(schema.contentDatabaseBodyHydrationQueue.sourceId, sourceId)); - await db - .delete(schema.contentDatabaseSourceExecutions) - .where(eq(schema.contentDatabaseSourceExecutions.sourceId, sourceId)); - await db - .delete(schema.contentDatabaseSourceChangeReviews) - .where(eq(schema.contentDatabaseSourceChangeReviews.sourceId, sourceId)); - await db - .delete(schema.contentDatabaseSourceChangeSets) - .where(eq(schema.contentDatabaseSourceChangeSets.sourceId, sourceId)); - await db - .delete(schema.contentDatabaseSourceRows) - .where(eq(schema.contentDatabaseSourceRows.sourceId, sourceId)); - await db - .delete(schema.contentDatabaseSourceFields) - .where(eq(schema.contentDatabaseSourceFields.sourceId, sourceId)); - await db - .delete(schema.contentDatabaseSources) - .where(eq(schema.contentDatabaseSources.id, sourceId)); + await db.transaction(async (tx) => { + await lockContentDatabaseMutation( + tx as unknown as ReturnType, + databaseId, + ); + const [source] = await tx + .select({ id: schema.contentDatabaseSources.id }) + .from(schema.contentDatabaseSources) + .where( + and( + eq(schema.contentDatabaseSources.id, sourceId), + eq(schema.contentDatabaseSources.databaseId, databaseId), + ), + ); + if (!source) return; + await tx + .delete(schema.contentDatabaseBodyHydrationQueue) + .where(eq(schema.contentDatabaseBodyHydrationQueue.sourceId, sourceId)); + await tx + .delete(schema.contentDatabaseSourceExecutions) + .where(eq(schema.contentDatabaseSourceExecutions.sourceId, sourceId)); + await tx + .delete(schema.contentDatabaseSourceChangeReviews) + .where(eq(schema.contentDatabaseSourceChangeReviews.sourceId, sourceId)); + await tx + .delete(schema.contentDatabaseSourceChangeSets) + .where(eq(schema.contentDatabaseSourceChangeSets.sourceId, sourceId)); + await tx + .delete(schema.contentDatabaseSourceRows) + .where(eq(schema.contentDatabaseSourceRows.sourceId, sourceId)); + await tx + .delete(schema.contentDatabaseSourceFields) + .where(eq(schema.contentDatabaseSourceFields.sourceId, sourceId)); + await tx + .delete(schema.contentDatabaseSources) + .where(eq(schema.contentDatabaseSources.id, sourceId)); + }); } export default defineAction({ @@ -70,13 +87,13 @@ export default defineAction({ eq(schema.contentDatabaseSources.databaseId, database.id), ), ); - if (target) await deleteSourceRecords(target.id); + if (target) await deleteSourceRecords(database.id, target.id); return getContentDatabaseResponse(database.id, { limit: 100, offset: 0 }); } const source = await getExistingSource(database.id); if (source) { - await deleteSourceRecords(source.id); + await deleteSourceRecords(database.id, source.id); } return getContentDatabaseResponse(database.id, { limit: 100, offset: 0 }); diff --git a/templates/content/actions/duplicate-database-item.ts b/templates/content/actions/duplicate-database-item.ts index 2168b1e569..a5eff57fe1 100644 --- a/templates/content/actions/duplicate-database-item.ts +++ b/templates/content/actions/duplicate-database-item.ts @@ -6,6 +6,10 @@ import { and, eq, gte, isNull, sql } from "drizzle-orm"; import { z } from "zod"; import { getDb, schema } from "../server/db/index.js"; +import { + lockContentDatabaseMutation, + touchContentDatabase, +} from "./_content-database-mutation-lock.js"; import { ensureDocumentFilesMembership } from "./_content-files.js"; import { assertNotWorkspaceCatalogDocuments } from "./_content-space-catalog-guards.js"; import { getContentDatabaseResponse } from "./_database-utils.js"; @@ -69,15 +73,6 @@ export default defineAction({ const now = new Date().toISOString(); const nextDocumentId = nanoid(); const nextItemId = nanoid(); - const nextTitle = - title?.trim() || `Copy of ${row.document.title.trim() || "Untitled"}`; - const nextPosition = row.item.position + 1; - - const values = await db - .select() - .from(schema.documentPropertyValues) - .where(eq(schema.documentPropertyValues.documentId, row.document.id)); - const inheritedShares = await db .select({ principalType: schema.documentShares.principalType, @@ -88,18 +83,52 @@ export default defineAction({ .where(eq(schema.documentShares.resourceId, row.database.documentId)); await db.transaction(async (tx) => { - const [lockedDatabase] = await tx - .update(schema.contentDatabases) - .set({ updatedAt: sql`${schema.contentDatabases.updatedAt}` }) + await lockContentDatabaseMutation( + tx as unknown as ReturnType, + row.database.id, + ); + await touchContentDatabase( + tx as unknown as ReturnType, + row.database.id, + now, + ); + const [lockedRow] = await tx + .select({ + item: schema.contentDatabaseItems, + document: schema.documents, + }) + .from(schema.contentDatabaseItems) + .innerJoin( + schema.documents, + eq(schema.documents.id, schema.contentDatabaseItems.documentId), + ) .where( and( - eq(schema.contentDatabases.id, row.database.id), - eq(schema.contentDatabases.ownerEmail, row.database.ownerEmail), - isNull(schema.contentDatabases.deletedAt), + eq(schema.contentDatabaseItems.id, row.item.id), + eq(schema.contentDatabaseItems.databaseId, row.database.id), + eq(schema.contentDatabaseItems.documentId, row.document.id), + isNull(schema.documents.trashedAt), ), - ) - .returning({ id: schema.contentDatabases.id }); - if (!lockedDatabase) throw new Error("Database is no longer active."); + ); + if (!lockedRow) { + throw new Error("Database row changed while duplication was waiting."); + } + if (lockedRow.document.spaceId !== row.database.spaceId) { + throw new Error( + "Cannot duplicate a database row across Content spaces.", + ); + } + + const nextTitle = + title?.trim() || + `Copy of ${lockedRow.document.title.trim() || "Untitled"}`; + const nextPosition = lockedRow.item.position + 1; + const values = await tx + .select() + .from(schema.documentPropertyValues) + .where( + eq(schema.documentPropertyValues.documentId, lockedRow.document.id), + ); const [claimedSource] = await tx .select({ id: schema.contentDatabaseItemKeyClaims.id }) .from(schema.contentDatabaseItemKeyClaims) @@ -123,7 +152,10 @@ export default defineAction({ }) .where( and( - eq(schema.contentDatabaseItems.databaseId, row.item.databaseId), + eq( + schema.contentDatabaseItems.databaseId, + lockedRow.item.databaseId, + ), gte(schema.contentDatabaseItems.position, nextPosition), ), ); @@ -136,7 +168,7 @@ export default defineAction({ }) .where( and( - eq(schema.documents.ownerEmail, row.document.ownerEmail), + eq(schema.documents.ownerEmail, lockedRow.document.ownerEmail), eq(schema.documents.parentId, row.database.documentId), gte(schema.documents.position, nextPosition), ), @@ -145,25 +177,25 @@ export default defineAction({ await tx.insert(schema.documents).values({ id: nextDocumentId, spaceId: row.database.spaceId, - ownerEmail: row.document.ownerEmail, - orgId: row.document.orgId, + ownerEmail: lockedRow.document.ownerEmail, + orgId: lockedRow.document.orgId, parentId: row.database.documentId, title: nextTitle, - content: row.document.content, - icon: row.document.icon, + content: lockedRow.document.content, + icon: lockedRow.document.icon, position: nextPosition, isFavorite: 0, - hideFromSearch: row.document.hideFromSearch, - visibility: row.document.visibility, + hideFromSearch: lockedRow.document.hideFromSearch, + visibility: lockedRow.document.visibility, createdAt: now, updatedAt: now, }); await tx.insert(schema.contentDatabaseItems).values({ id: nextItemId, - ownerEmail: row.item.ownerEmail, - orgId: row.item.orgId, - databaseId: row.item.databaseId, + ownerEmail: lockedRow.item.ownerEmail, + orgId: lockedRow.item.orgId, + databaseId: lockedRow.item.databaseId, documentId: nextDocumentId, position: nextPosition, createdAt: now, @@ -178,7 +210,7 @@ export default defineAction({ principalType: share.principalType, principalId: share.principalId, role: share.role, - createdBy: getRequestUserEmail() ?? row.document.ownerEmail, + createdBy: getRequestUserEmail() ?? lockedRow.document.ownerEmail, createdAt: now, })), ); @@ -188,7 +220,7 @@ export default defineAction({ await tx.insert(schema.documentPropertyValues).values( values.map((value) => ({ id: nanoid(), - ownerEmail: row.document.ownerEmail, + ownerEmail: lockedRow.document.ownerEmail, documentId: nextDocumentId, propertyId: value.propertyId, valueJson: value.valueJson, diff --git a/templates/content/actions/duplicate-database-items.ts b/templates/content/actions/duplicate-database-items.ts index 3ef6456dc6..63b73f1534 100644 --- a/templates/content/actions/duplicate-database-items.ts +++ b/templates/content/actions/duplicate-database-items.ts @@ -2,9 +2,13 @@ import { defineAction } from "@agent-native/core"; import { writeAppState } from "@agent-native/core/application-state"; import { getRequestUserEmail } from "@agent-native/core/server/request-context"; import { assertAccess } from "@agent-native/core/sharing"; -import { and, eq, gte, inArray, isNull, sql } from "drizzle-orm"; +import { and, asc, eq, gte, inArray, isNull, sql } from "drizzle-orm"; import { getDb, schema } from "../server/db/index.js"; +import { + lockContentDatabaseMutation, + touchContentDatabase, +} from "./_content-database-mutation-lock.js"; import { ensureDocumentsFilesMembership } from "./_content-files.js"; import { assertNotWorkspaceCatalogDocuments } from "./_content-space-catalog-guards.js"; import { @@ -41,32 +45,8 @@ export default defineAction({ ); const sourceItemIds = rows.map((row) => row.item.id); const now = new Date().toISOString(); - const insertionPosition = - Math.max(...rows.map((row) => row.item.position)) + 1; const currentUserEmail = getRequestUserEmail() ?? database.ownerEmail; - const values = - sourceDocumentIds.length > 0 - ? await db - .select() - .from(schema.documentPropertyValues) - .where( - inArray( - schema.documentPropertyValues.documentId, - sourceDocumentIds, - ), - ) - : []; - const valuesByDocumentId = new Map< - string, - Array - >(); - for (const value of values) { - const list = valuesByDocumentId.get(value.documentId) ?? []; - list.push(value); - valuesByDocumentId.set(value.documentId, list); - } - const inheritedShares = await db .select({ principalType: schema.documentShares.principalType, @@ -76,28 +56,63 @@ export default defineAction({ .from(schema.documentShares) .where(eq(schema.documentShares.resourceId, database.documentId)); - const duplicates = rows.map((row, index) => ({ + const duplicates = rows.map((row) => ({ sourceItemId: row.item.id, sourceDocumentId: row.document.id, duplicatedItemId: nanoid(), duplicatedDocumentId: nanoid(), - position: insertionPosition + index, - row, })); await db.transaction(async (tx) => { - const [lockedDatabase] = await tx - .update(schema.contentDatabases) - .set({ updatedAt: sql`${schema.contentDatabases.updatedAt}` }) + await lockContentDatabaseMutation( + tx as unknown as ReturnType, + database.id, + ); + await touchContentDatabase( + tx as unknown as ReturnType, + database.id, + now, + ); + const lockedRows = await tx + .select({ + item: schema.contentDatabaseItems, + document: schema.documents, + }) + .from(schema.contentDatabaseItems) + .innerJoin( + schema.documents, + eq(schema.documents.id, schema.contentDatabaseItems.documentId), + ) .where( and( - eq(schema.contentDatabases.id, database.id), - eq(schema.contentDatabases.ownerEmail, database.ownerEmail), - isNull(schema.contentDatabases.deletedAt), + eq(schema.contentDatabaseItems.databaseId, database.id), + inArray(schema.contentDatabaseItems.id, sourceItemIds), + isNull(schema.documents.trashedAt), ), ) - .returning({ id: schema.contentDatabases.id }); - if (!lockedDatabase) throw new Error("Database is no longer active."); + .orderBy(asc(schema.contentDatabaseItems.position)); + const lockedRowsByItemId = new Map( + lockedRows.map((lockedRow) => [lockedRow.item.id, lockedRow]), + ); + if ( + lockedRows.length !== rows.length || + rows.some( + (row) => + lockedRowsByItemId.get(row.item.id)?.document.id !== + row.document.id, + ) + ) { + throw new Error("Database rows changed while duplication was waiting."); + } + if ( + lockedRows.some( + (lockedRow) => lockedRow.document.spaceId !== database.spaceId, + ) + ) { + throw new Error( + "Cannot duplicate database rows across Content spaces.", + ); + } const [claimedSource] = await tx .select({ id: schema.contentDatabaseItemKeyClaims.id }) .from(schema.contentDatabaseItemKeyClaims) @@ -116,10 +131,38 @@ export default defineAction({ "Rows with active stable-key claims cannot be duplicated.", ); } + const insertionPosition = + Math.max(...lockedRows.map((lockedRow) => lockedRow.item.position)) + 1; + const lockedDuplicates = duplicates.map((duplicate, index) => ({ + ...duplicate, + position: insertionPosition + index, + row: lockedRowsByItemId.get(duplicate.sourceItemId)!, + })); + const values = + sourceDocumentIds.length > 0 + ? await tx + .select() + .from(schema.documentPropertyValues) + .where( + inArray( + schema.documentPropertyValues.documentId, + sourceDocumentIds, + ), + ) + : []; + const valuesByDocumentId = new Map< + string, + Array + >(); + for (const value of values) { + const list = valuesByDocumentId.get(value.documentId) ?? []; + list.push(value); + valuesByDocumentId.set(value.documentId, list); + } await tx .update(schema.contentDatabaseItems) .set({ - position: sql`${schema.contentDatabaseItems.position} + ${duplicates.length}`, + position: sql`${schema.contentDatabaseItems.position} + ${lockedDuplicates.length}`, updatedAt: now, }) .where( @@ -132,7 +175,7 @@ export default defineAction({ await tx .update(schema.documents) .set({ - position: sql`${schema.documents.position} + ${duplicates.length}`, + position: sql`${schema.documents.position} + ${lockedDuplicates.length}`, updatedAt: now, }) .where( @@ -144,7 +187,7 @@ export default defineAction({ ); await tx.insert(schema.documents).values( - duplicates.map((duplicate) => ({ + lockedDuplicates.map((duplicate) => ({ id: duplicate.duplicatedDocumentId, spaceId: database.spaceId, ownerEmail: duplicate.row.document.ownerEmail, @@ -163,7 +206,7 @@ export default defineAction({ ); await tx.insert(schema.contentDatabaseItems).values( - duplicates.map((duplicate) => ({ + lockedDuplicates.map((duplicate) => ({ id: duplicate.duplicatedItemId, ownerEmail: duplicate.row.item.ownerEmail, orgId: duplicate.row.item.orgId, @@ -175,7 +218,7 @@ export default defineAction({ })), ); - const duplicatedValues = duplicates.flatMap((duplicate) => + const duplicatedValues = lockedDuplicates.flatMap((duplicate) => (valuesByDocumentId.get(duplicate.sourceDocumentId) ?? []).map( (value) => ({ id: nanoid(), @@ -194,7 +237,7 @@ export default defineAction({ if (inheritedShares.length > 0) { await tx.insert(schema.documentShares).values( - duplicates.flatMap((duplicate) => + lockedDuplicates.flatMap((duplicate) => inheritedShares.map((share) => ({ id: nanoid(), resourceId: duplicate.duplicatedDocumentId, @@ -209,7 +252,7 @@ export default defineAction({ } await ensureDocumentsFilesMembership( tx, - duplicates.map((duplicate) => duplicate.duplicatedDocumentId), + lockedDuplicates.map((duplicate) => duplicate.duplicatedDocumentId), now, ); }); diff --git a/templates/content/actions/duplicate-document-property.ts b/templates/content/actions/duplicate-document-property.ts index 864712e59f..d61759ee11 100644 --- a/templates/content/actions/duplicate-document-property.ts +++ b/templates/content/actions/duplicate-document-property.ts @@ -10,6 +10,7 @@ import { serializePropertyOptions, type DocumentPropertyType, } from "../shared/properties.js"; +import { lockContentDatabaseMutation } from "./_content-database-mutation-lock.js"; import { lockDatabaseMemberships } from "./_database-membership-lock.js"; import { propertyDefinitionsPositionScope, @@ -65,19 +66,14 @@ export default defineAction({ const now = new Date().toISOString(); const newPropertyId = nanoid(); - const isBlocks = isBlocksPropertyType( - definition.type as DocumentPropertyType, - ); - // A duplicated Blocks field is a brand-new, independent, EMPTY field — never - // primary (only one field backs the body) and with no copied content. - const optionsJson = isBlocks - ? serializePropertyOptions({ blocks: { primary: false } }) - : definition.optionsJson; - await withPositionLock( propertyDefinitionsPositionScope(database.id), async () => { await db.transaction(async (tx) => { + await lockContentDatabaseMutation( + tx as unknown as ReturnType, + database.id, + ); const memberships = await tx .select({ id: schema.contentDatabaseItems.id }) .from(schema.contentDatabaseItems) @@ -86,6 +82,33 @@ export default defineAction({ tx, memberships.map((membership) => membership.id), ); + const [lockedDefinition] = await tx + .select() + .from(schema.documentPropertyDefinitions) + .where( + and( + eq(schema.documentPropertyDefinitions.id, propertyId), + eq( + schema.documentPropertyDefinitions.ownerEmail, + document.ownerEmail, + ), + eq(schema.documentPropertyDefinitions.databaseId, database.id), + ), + ); + if (!lockedDefinition) { + throw new Error(`Property "${propertyId}" not found`); + } + if (lockedDefinition.systemRole) { + throw new Error("System properties cannot be duplicated."); + } + const isBlocks = isBlocksPropertyType( + lockedDefinition.type as DocumentPropertyType, + ); + // A duplicated Blocks field is a brand-new, independent, EMPTY field — never + // primary (only one field backs the body) and with no copied content. + const optionsJson = isBlocks + ? serializePropertyOptions({ blocks: { primary: false } }) + : lockedDefinition.optionsJson; const [maxPos] = await tx .select({ @@ -104,12 +127,12 @@ export default defineAction({ await tx.insert(schema.documentPropertyDefinitions).values({ id: newPropertyId, - ownerEmail: definition.ownerEmail, - orgId: definition.orgId, + ownerEmail: lockedDefinition.ownerEmail, + orgId: lockedDefinition.orgId, databaseId: database.id, - name: `${definition.name} copy`, - type: definition.type, - visibility: definition.visibility, + name: `${lockedDefinition.name} copy`, + type: lockedDefinition.type, + visibility: lockedDefinition.visibility, optionsJson, position: (maxPos?.max ?? -1) + 1, createdAt: now, diff --git a/templates/content/actions/migrate-content-database-rows.db.test.ts b/templates/content/actions/migrate-content-database-rows.db.test.ts new file mode 100644 index 0000000000..9ba96b6f6b --- /dev/null +++ b/templates/content/actions/migrate-content-database-rows.db.test.ts @@ -0,0 +1,1306 @@ +import { existsSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import * as coreDb from "@agent-native/core/db"; +import { runWithRequestContext } from "@agent-native/core/server"; +import { and, eq, inArray, sql } from "drizzle-orm"; +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; + +const durableLock = vi.hoisted(() => ({ entered: false })); +const flushOpenDocumentEditorToSql = vi.hoisted(() => vi.fn()); + +vi.mock("./_document-flush.js", () => ({ + flushOpenDocumentEditorToSql, +})); + +vi.mock("./_content-database-mutation-lock.js", async (importOriginal) => { + const original = + await importOriginal< + typeof import("./_content-database-mutation-lock.js") + >(); + return { + ...original, + lockContentDatabaseMutation: async ( + ...args: Parameters + ) => { + durableLock.entered = true; + return original.lockContentDatabaseMutation(...args); + }, + }; +}); + +const TEST_DB_PATH = join( + tmpdir(), + `content-database-row-migration-${process.pid}-${Date.now()}.sqlite`, +); +const PGLITE_DB_PATH = `${TEST_DB_PATH}.pglite`; +const TEST_DATABASE_URL = + process.env.CONTENT_MIGRATION_TEST_BACKEND === "pglite" + ? `pglite:${PGLITE_DB_PATH}` + : `file:${TEST_DB_PATH}`; +const OWNER = "synthetic-migration-owner@example.test"; +const OUTSIDER = "synthetic-migration-outsider@example.test"; +let getDb: () => any; +let schema: typeof import("../server/db/schema.js"); +let action: typeof import("./migrate-content-database-rows.js").default; +let lockContentDatabaseMutation: typeof import("./_content-database-mutation-lock.js").lockContentDatabaseMutation; +let touchContentDatabase: typeof import("./_content-database-mutation-lock.js").touchContentDatabase; +let serializeMigrationValue: typeof import("./_content-database-row-migration.js").serializeMigrationValue; +const now = () => new Date().toISOString(); + +beforeAll(async () => { + process.env.DATABASE_URL = TEST_DATABASE_URL; + const database = await import("../server/db/index.js"); + getDb = database.getDb; + schema = database.schema; + serializeMigrationValue = ( + await import("./_content-database-row-migration.js") + ).serializeMigrationValue; + action = (await import("./migrate-content-database-rows.js")).default; + ({ lockContentDatabaseMutation, touchContentDatabase } = + await import("./_content-database-mutation-lock.js")); + await (await import("../server/plugins/db.js")).default(undefined as any); +}, 60_000); + +beforeEach(() => { + vi.restoreAllMocks(); + durableLock.entered = false; + flushOpenDocumentEditorToSql.mockReset(); + flushOpenDocumentEditorToSql.mockResolvedValue(undefined); +}); + +afterAll(() => { + if (TEST_DATABASE_URL.startsWith("file:")) { + for (const suffix of ["", "-wal", "-shm"]) + rmSync(`${TEST_DB_PATH}${suffix}`, { force: true }); + for (const suffix of ["", "-wal", "-shm"]) + expect(existsSync(`${TEST_DB_PATH}${suffix}`)).toBe(false); + } + if (TEST_DATABASE_URL.startsWith("pglite:")) { + rmSync(PGLITE_DB_PATH, { force: true, recursive: true }); + expect(existsSync(PGLITE_DB_PATH)).toBe(false); + } +}); + +async function fixture(rowCount = 20) { + const db = getDb(); + const stamp = now(); + const key = `${Date.now()}_${Math.random().toString(36).slice(2)}`; + const databaseId = `synthetic_migration_db_${key}`; + const databaseDocumentId = `synthetic_migration_page_${key}`; + await db.insert(schema.documents).values({ + id: databaseDocumentId, + ownerEmail: OWNER, + spaceId: "synthetic_space", + title: "Synthetic migration database", + content: "", + visibility: "private", + createdAt: stamp, + updatedAt: stamp, + }); + await db.insert(schema.contentDatabases).values({ + id: databaseId, + ownerEmail: OWNER, + spaceId: "synthetic_space", + documentId: databaseDocumentId, + title: "Synthetic migration database", + createdAt: stamp, + updatedAt: stamp, + }); + const definitions = [ + { id: `status_${key}`, name: "Status", type: "status", systemRole: null }, + { id: `cluster_${key}`, name: "Cluster", type: "text", systemRole: null }, + { id: `evidence_${key}`, name: "Evidence", type: "url", systemRole: null }, + ]; + await db.insert(schema.documentPropertyDefinitions).values( + definitions.map((definition, position) => ({ + ...definition, + ownerEmail: OWNER, + databaseId, + visibility: "always_show", + optionsJson: "{}", + position, + createdAt: stamp, + updatedAt: stamp, + })), + ); + const rows = [] as any[]; + for (let index = 0; index < rowCount; index += 1) { + const documentId = `synthetic_row_doc_${key}_${index}`; + const itemId = `synthetic_row_item_${key}_${index}`; + const updatedAt = `2026-01-01T00:00:${String(index).padStart(2, "0")}.000Z`; + const status = ["open", "in_progress", "closed"][index % 3]; + await db.insert(schema.documents).values({ + id: documentId, + ownerEmail: OWNER, + spaceId: "synthetic_space", + parentId: databaseDocumentId, + title: `Synthetic ${index}`, + content: `# Synthetic heading ${index % 5}`, + visibility: "private", + hideFromSearch: 1, + position: index, + createdAt: stamp, + updatedAt, + }); + await db.insert(schema.contentDatabaseItems).values({ + id: itemId, + ownerEmail: OWNER, + databaseId, + documentId, + position: index, + createdAt: stamp, + updatedAt: stamp, + }); + await db.insert(schema.documentPropertyValues).values([ + { + id: `synthetic_status_${key}_${index}`, + ownerEmail: OWNER, + documentId, + propertyId: definitions[0].id, + valueJson: JSON.stringify(status), + createdAt: stamp, + updatedAt: stamp, + }, + { + id: `synthetic_cluster_${key}_${index}`, + ownerEmail: OWNER, + documentId, + propertyId: definitions[1].id, + valueJson: '"blue"', + createdAt: stamp, + updatedAt: stamp, + }, + { + id: `synthetic_evidence_${key}_${index}`, + ownerEmail: OWNER, + documentId, + propertyId: definitions[2].id, + valueJson: `"https://synthetic.example.test/evidence/${index}"`, + createdAt: stamp, + updatedAt: stamp, + }, + ]); + rows.push({ + itemId, + documentId, + expectedUpdatedAt: updatedAt, + content: `# User need\nSynthetic need ${index}\n\n# Slack context\nSynthetic context ${index}\n\n# Assessment\nSynthetic assessment ${index}\n\n# Implementation evidence\nSynthetic evidence ${index}\n\n# Remaining gap\nSynthetic gap ${index}`, + propertyValues: [ + { + propertyId: `reported_by_${key}`, + value: `Synthetic person ${index}`, + }, + { + propertyId: `slack_thread_${key}`, + value: `https://synthetic.example.test/slack/${index}`, + }, + { + propertyId: `reported_date_${key}`, + value: { start: "2026-01-01" }, + }, + { + propertyId: `roadmap_feature_${key}`, + value: + index % 2 === 0 + ? [ + "content.feature.durable-foundations", + "content.feature.living-references", + ] + : ["content.feature.durable-foundations"], + }, + ], + protectedPropertyValues: [ + { propertyId: definitions[0].id, valueJson: JSON.stringify(status) }, + ], + }); + } + return { databaseId, databaseDocumentId, key, rows, definitions }; +} + +function plan(seed: Awaited>) { + return { + databaseId: seed.databaseId, + databaseDocumentId: seed.databaseDocumentId, + idempotencyKey: `synthetic-key-${seed.key}`, + expectedRowCount: seed.rows.length, + legacyPropertyIds: [seed.definitions[1].id, seed.definitions[2].id], + propertyDefinitions: [ + { + id: `reported_by_${seed.key}`, + name: "Reported by", + type: "text", + visibility: "always_show", + }, + { + id: `slack_thread_${seed.key}`, + name: "Slack thread", + type: "url", + visibility: "always_show", + }, + { + id: `reported_date_${seed.key}`, + name: "Reported date", + type: "date", + visibility: "hide_when_empty", + }, + { + id: `roadmap_feature_${seed.key}`, + name: "Roadmap feature", + type: "multi_select", + visibility: "always_show", + options: [ + { + id: "content.feature.durable-foundations", + name: "Durable foundations", + color: "blue", + }, + { + id: "content.feature.living-references", + name: "Living references", + color: "green", + }, + ], + }, + ], + rows: seed.rows, + } as const; +} + +async function readFixtureState(seed: Awaited>) { + const db = getDb(); + const documentIds = seed.rows.map((row: any) => row.documentId); + const byId = (left: any, right: any) => left.id.localeCompare(right.id); + const documents = await db + .select() + .from(schema.documents) + .where(inArray(schema.documents.id, documentIds)); + const items = await db + .select() + .from(schema.contentDatabaseItems) + .where(eq(schema.contentDatabaseItems.databaseId, seed.databaseId)); + const definitions = await db + .select() + .from(schema.documentPropertyDefinitions) + .where(eq(schema.documentPropertyDefinitions.databaseId, seed.databaseId)); + const values = await db + .select() + .from(schema.documentPropertyValues) + .where(inArray(schema.documentPropertyValues.documentId, documentIds)); + const shares = await db + .select() + .from(schema.documentShares) + .where( + inArray(schema.documentShares.resourceId, [ + seed.databaseDocumentId, + ...documentIds, + ]), + ); + return { + documents: documents + .map(({ updatedAt: _updatedAt, ...document }: any) => document) + .sort(byId), + items: items + .map(({ updatedAt: _updatedAt, ...item }: any) => item) + .sort(byId), + definitions: definitions + .map(({ updatedAt: _updatedAt, ...definition }: any) => definition) + .sort(byId), + values: values + .map(({ updatedAt: _updatedAt, ...value }: any) => value) + .sort(byId), + shares: shares.sort(byId), + }; +} + +/** + * Terminal replays must be observationally inert. Keep timestamps and the + * receipt here: stripping them would hide a lock or receipt rewrite. + */ +async function readDurableMigrationState( + seed: Awaited>, +) { + const db = getDb(); + const documentIds = [ + seed.databaseDocumentId, + ...seed.rows.map((row: any) => row.documentId), + ]; + const byId = (left: any, right: any) => left.id.localeCompare(right.id); + const [databases, documents, items, definitions, values, versions, receipts] = + await Promise.all([ + db + .select() + .from(schema.contentDatabases) + .where(eq(schema.contentDatabases.id, seed.databaseId)), + db + .select() + .from(schema.documents) + .where(inArray(schema.documents.id, documentIds)), + db + .select() + .from(schema.contentDatabaseItems) + .where(eq(schema.contentDatabaseItems.databaseId, seed.databaseId)), + db + .select() + .from(schema.documentPropertyDefinitions) + .where( + eq(schema.documentPropertyDefinitions.databaseId, seed.databaseId), + ), + db + .select() + .from(schema.documentPropertyValues) + .where(inArray(schema.documentPropertyValues.documentId, documentIds)), + db + .select() + .from(schema.documentVersions) + .where(inArray(schema.documentVersions.documentId, documentIds)), + db + .select() + .from(schema.contentDatabaseMigrationReceipts) + .where( + eq( + schema.contentDatabaseMigrationReceipts.databaseId, + seed.databaseId, + ), + ), + ]); + return { + databases: databases.sort(byId), + documents: documents.sort(byId), + items: items.sort(byId), + definitions: definitions.sort(byId), + values: values.sort(byId), + versions: versions.sort(byId), + receipts: receipts.sort(byId), + }; +} + +describe("migrate-content-database-rows", () => { + it.each(["apply", "rollback"] as const)( + "flushes local editor state before acquiring the durable lock during %s", + async (phase) => { + const seed = await fixture(1); + const input = plan(seed); + const applied = + phase === "rollback" + ? await runWithRequestContext({ userEmail: OWNER }, () => + action.run({ phase: "apply", plan: input }), + ) + : null; + durableLock.entered = false; + flushOpenDocumentEditorToSql.mockClear(); + flushOpenDocumentEditorToSql.mockImplementation(async () => { + expect(durableLock.entered).toBe(false); + }); + + await runWithRequestContext({ userEmail: OWNER }, () => + phase === "apply" + ? action.run({ phase: "apply", plan: input }) + : action.run({ + phase: "rollback", + databaseId: seed.databaseId, + idempotencyKey: input.idempotencyKey, + expectedPostDigest: applied!.postDigest, + }), + ); + + expect(flushOpenDocumentEditorToSql).toHaveBeenCalledOnce(); + expect(durableLock.entered).toBe(true); + }, + ); + + it("rejects a local editor change on the post-flush reload", async () => { + const seed = await fixture(1); + const input = plan(seed); + flushOpenDocumentEditorToSql.mockImplementationOnce( + async ({ documentId }: { documentId: string }) => { + await getDb() + .update(schema.documents) + .set({ + content: "# Saved during flush", + updatedAt: "2026-01-01T00:00:01.000Z", + }) + .where(eq(schema.documents.id, documentId)); + }, + ); + + await expect( + runWithRequestContext({ userEmail: OWNER }, () => + action.run({ phase: "apply", plan: input }), + ), + ).rejects.toThrow("Stale row"); + expect(durableLock.entered).toBe(true); + expect( + await getDb() + .select() + .from(schema.contentDatabaseMigrationReceipts) + .where( + eq( + schema.contentDatabaseMigrationReceipts.databaseId, + seed.databaseId, + ), + ), + ).toHaveLength(0); + }); + + it("drains every local editor flush before reporting a failure", async () => { + const seed = await fixture(2); + const input = plan(seed); + let releaseSecond = () => {}; + const secondReleased = new Promise((resolve) => { + releaseSecond = resolve; + }); + let secondEntered = () => {}; + const secondStarted = new Promise((resolve) => { + secondEntered = resolve; + }); + flushOpenDocumentEditorToSql + .mockRejectedValueOnce(new Error("Synthetic flush failure")) + .mockImplementationOnce(async () => { + secondEntered(); + await secondReleased; + }); + + let settled = false; + const operation = runWithRequestContext({ userEmail: OWNER }, () => + action.run({ phase: "apply", plan: input }), + ).finally(() => { + settled = true; + }); + await secondStarted; + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(settled).toBe(false); + expect(durableLock.entered).toBe(false); + releaseSecond(); + await expect(operation).rejects.toThrow("Synthetic flush failure"); + }); + + it("fails closed on shared SQLite before flushing but permits a terminal replay", async () => { + const seed = await fixture(1); + const input = plan(seed); + const applied = await runWithRequestContext({ userEmail: OWNER }, () => + action.run({ phase: "apply", plan: input }), + ); + const beforeReplay = await readDurableMigrationState(seed); + const localSpy = vi.spyOn(coreDb, "isLocalDatabase").mockReturnValue(false); + const postgresSpy = vi.spyOn(coreDb, "isPostgres").mockReturnValue(false); + try { + flushOpenDocumentEditorToSql.mockClear(); + const replayed = await runWithRequestContext({ userEmail: OWNER }, () => + action.run({ phase: "apply", plan: input }), + ); + expect(replayed).toMatchObject({ + receiptId: applied.receiptId, + replayed: true, + }); + expect(flushOpenDocumentEditorToSql).not.toHaveBeenCalled(); + expect(await readDurableMigrationState(seed)).toEqual(beforeReplay); + + const fresh = await fixture(1); + await expect( + runWithRequestContext({ userEmail: OWNER }, () => + action.run({ phase: "apply", plan: plan(fresh) }), + ), + ).rejects.toThrow( + "requires PostgreSQL or a local SQLite/PGlite database", + ); + expect(flushOpenDocumentEditorToSql).not.toHaveBeenCalled(); + } finally { + localSpy.mockRestore(); + postgresSpy.mockRestore(); + } + }); + + it("validates without writes, applies all 20 synthetic rows, and replays without versions", async () => { + const seed = await fixture(); + const input = plan(seed); + const db = getDb(); + const validated = await runWithRequestContext({ userEmail: OWNER }, () => + action.run({ phase: "validate", plan: input }), + ); + expect(validated).toMatchObject({ written: 0, counts: { rows: 20 } }); + expect( + await db + .select() + .from(schema.documentVersions) + .where( + inArray( + schema.documentVersions.documentId, + seed.rows.map((row: any) => row.documentId), + ), + ), + ).toHaveLength(0); + const applied = await runWithRequestContext({ userEmail: OWNER }, () => + action.run({ phase: "apply", plan: input }), + ); + expect(applied).toMatchObject({ + state: "applied", + written: 20, + replayed: false, + verified: false, + }); + const migrated = await db + .select() + .from(schema.documents) + .where( + inArray( + schema.documents.id, + seed.rows.map((row: any) => row.documentId), + ), + ); + const items = await db + .select() + .from(schema.contentDatabaseItems) + .where(eq(schema.contentDatabaseItems.databaseId, seed.databaseId)); + const values = await db + .select() + .from(schema.documentPropertyValues) + .where( + inArray( + schema.documentPropertyValues.documentId, + seed.rows.map((row: any) => row.documentId), + ), + ); + const shares = await db + .select() + .from(schema.documentShares) + .where( + inArray(schema.documentShares.resourceId, [ + seed.databaseDocumentId, + ...seed.rows.map((row: any) => row.documentId), + ]), + ); + const migratedDefinitions = await db + .select() + .from(schema.documentPropertyDefinitions) + .where( + eq(schema.documentPropertyDefinitions.databaseId, seed.databaseId), + ); + expect(migrated).toHaveLength(20); + expect(items).toHaveLength(20); + expect(shares).toHaveLength(0); + for (const expected of input.propertyDefinitions) { + expect( + migratedDefinitions.find( + (definition: any) => definition.id === expected.id, + ), + ).toMatchObject({ + id: expected.id, + ownerEmail: OWNER, + databaseId: seed.databaseId, + name: expected.name, + type: expected.type, + visibility: expected.visibility, + optionsJson: JSON.stringify( + expected.type === "multi_select" ? { options: expected.options } : {}, + ), + }); + } + for (const expected of seed.rows) { + const document = migrated.find( + (candidate: any) => candidate.id === expected.documentId, + ); + const item = items.find( + (candidate: any) => candidate.id === expected.itemId, + ); + expect(document).toMatchObject({ + id: expected.documentId, + ownerEmail: OWNER, + spaceId: "synthetic_space", + parentId: seed.databaseDocumentId, + content: expected.content, + visibility: "private", + hideFromSearch: 1, + }); + expect(item).toMatchObject({ + id: expected.itemId, + databaseId: seed.databaseId, + documentId: expected.documentId, + ownerEmail: OWNER, + }); + const rowValues = values.filter( + (value: any) => value.documentId === expected.documentId, + ); + expect(rowValues).toHaveLength(7); + expect( + rowValues.find( + (value: any) => value.propertyId === seed.definitions[0].id, + )?.valueJson, + ).toBe(expected.protectedPropertyValues[0].valueJson); + for (const propertyValue of expected.propertyValues) { + const definition = input.propertyDefinitions.find( + (candidate) => candidate.id === propertyValue.propertyId, + )!; + expect( + rowValues.find( + (value: any) => value.propertyId === propertyValue.propertyId, + )?.valueJson, + ).toBe(serializeMigrationValue(definition, propertyValue.value)); + } + } + const versions = await db + .select() + .from(schema.documentVersions) + .where( + inArray( + schema.documentVersions.documentId, + seed.rows.map((row: any) => row.documentId), + ), + ); + expect(versions).toHaveLength(20); + const replayed = await runWithRequestContext({ userEmail: OWNER }, () => + action.run({ phase: "apply", plan: input }), + ); + expect(replayed).toMatchObject({ replayed: true, state: "applied" }); + expect( + await db + .select() + .from(schema.documentVersions) + .where( + inArray( + schema.documentVersions.documentId, + seed.rows.map((row: any) => row.documentId), + ), + ), + ).toHaveLength(20); + }); + + it("refuses unauthorised, stale, and unknown-option plans before a write", async () => { + const seed = await fixture(); + const input: any = structuredClone(plan(seed)); + const before = await readFixtureState(seed); + const originalTimestamp = seed.rows[0].expectedUpdatedAt; + await expect( + runWithRequestContext({ userEmail: OUTSIDER }, () => + action.run({ phase: "apply", plan: input }), + ), + ).rejects.toThrow(); + input.rows[0].expectedUpdatedAt = "stale"; + await expect( + runWithRequestContext({ userEmail: OWNER }, () => + action.run({ phase: "apply", plan: input }), + ), + ).rejects.toThrow("Stale row"); + input.rows[0].expectedUpdatedAt = originalTimestamp; + input.rows[0].propertyValues[3].value = ["unknown"]; + await expect( + runWithRequestContext({ userEmail: OWNER }, () => + action.run({ phase: "apply", plan: input }), + ), + ).rejects.toThrow("Unknown multi-select option"); + expect(await readFixtureState(seed)).toEqual(before); + expect( + await getDb() + .select() + .from(schema.contentDatabaseMigrationReceipts) + .where( + eq( + schema.contentDatabaseMigrationReceipts.databaseId, + seed.databaseId, + ), + ), + ).toHaveLength(0); + }); + + it.skipIf(TEST_DATABASE_URL.startsWith("pglite:"))( + "rejects same-key changed plans and rolls all writes back on an abort trigger", + async () => { + const seed = await fixture(); + const input: any = structuredClone(plan(seed)); + const db = getDb(); + const before = await readFixtureState(seed); + await db.run( + sql.raw( + `CREATE TRIGGER synthetic_migration_abort BEFORE UPDATE ON documents WHEN NEW.id = '${seed.rows[8].documentId}' BEGIN SELECT RAISE(ABORT, 'synthetic migration abort'); END`, + ), + ); + await expect( + runWithRequestContext({ userEmail: OWNER }, () => + action.run({ phase: "apply", plan: input }), + ), + ).rejects.toThrow("synthetic migration abort"); + expect(await readFixtureState(seed)).toEqual(before); + expect( + await db + .select() + .from(schema.documentVersions) + .where( + inArray( + schema.documentVersions.documentId, + seed.rows.map((row: any) => row.documentId), + ), + ), + ).toHaveLength(0); + expect( + await db + .select() + .from(schema.documentPropertyDefinitions) + .where( + eq(schema.documentPropertyDefinitions.databaseId, seed.databaseId), + ), + ).toHaveLength(3); + expect( + await db + .select() + .from(schema.contentDatabaseMigrationReceipts) + .where( + eq( + schema.contentDatabaseMigrationReceipts.databaseId, + seed.databaseId, + ), + ), + ).toHaveLength(0); + await db.run(sql`DROP TRIGGER synthetic_migration_abort`); + const applied: any = await runWithRequestContext( + { userEmail: OWNER }, + () => action.run({ phase: "apply", plan: input }), + ); + const changed = structuredClone(input); + changed.rows[0].content = "# Different synthetic body"; + await expect( + runWithRequestContext({ userEmail: OWNER }, () => + action.run({ phase: "apply", plan: changed }), + ), + ).rejects.toThrow("different migration plan"); + expect(applied.state).toBe("applied"); + }, + ); + + it("serializes simultaneous same-key applies into one commit and one replay", async () => { + const seed = await fixture(); + const input = plan(seed); + const results = await Promise.all([ + runWithRequestContext({ userEmail: OWNER }, () => + action.run({ phase: "apply", plan: input }), + ), + runWithRequestContext({ userEmail: OWNER }, () => + action.run({ phase: "apply", plan: input }), + ), + ]); + + expect(results.map((result: any) => result.replayed).sort()).toEqual([ + false, + true, + ]); + expect( + await getDb() + .select() + .from(schema.contentDatabaseMigrationReceipts) + .where( + eq( + schema.contentDatabaseMigrationReceipts.databaseId, + seed.databaseId, + ), + ), + ).toHaveLength(1); + expect( + await getDb() + .select() + .from(schema.documentVersions) + .where( + inArray( + schema.documentVersions.documentId, + seed.rows.map((row: any) => row.documentId), + ), + ), + ).toHaveLength(seed.rows.length); + }); + + it("serializes an ordinary row addition against the exact migration snapshot", async () => { + const seed = await fixture(); + const input = plan(seed); + const concurrentDocumentId = `concurrent_document_${seed.key}`; + const [migration, addition] = await Promise.allSettled([ + runWithRequestContext({ userEmail: OWNER }, () => + action.run({ phase: "apply", plan: input }), + ), + getDb().transaction(async (tx: any) => { + const stamp = now(); + await lockContentDatabaseMutation(tx, seed.databaseId); + await touchContentDatabase(tx, seed.databaseId, stamp); + await tx.insert(schema.documents).values({ + id: concurrentDocumentId, + ownerEmail: OWNER, + spaceId: "synthetic_space", + parentId: seed.databaseDocumentId, + title: "Concurrent synthetic row", + content: "", + visibility: "private", + position: 20, + createdAt: stamp, + updatedAt: stamp, + }); + await tx.insert(schema.contentDatabaseItems).values({ + id: `concurrent_item_${seed.key}`, + ownerEmail: OWNER, + databaseId: seed.databaseId, + documentId: concurrentDocumentId, + position: 20, + createdAt: stamp, + updatedAt: stamp, + }); + }), + ]); + + if (addition.status === "rejected") throw addition.reason; + expect( + await getDb() + .select() + .from(schema.contentDatabaseItems) + .where(eq(schema.contentDatabaseItems.databaseId, seed.databaseId)), + ).toHaveLength(21); + const receipts = await getDb() + .select() + .from(schema.contentDatabaseMigrationReceipts) + .where( + eq(schema.contentDatabaseMigrationReceipts.databaseId, seed.databaseId), + ); + const newDefinitions = await getDb() + .select() + .from(schema.documentPropertyDefinitions) + .where( + inArray( + schema.documentPropertyDefinitions.id, + input.propertyDefinitions.map((definition) => definition.id), + ), + ); + const [firstRow] = await getDb() + .select({ content: schema.documents.content }) + .from(schema.documents) + .where(eq(schema.documents.id, seed.rows[0].documentId)); + if (migration.status === "fulfilled") { + expect(receipts).toHaveLength(1); + expect(newDefinitions).toHaveLength(input.propertyDefinitions.length); + expect(firstRow.content).toBe(input.rows[0].content); + } else { + expect(receipts).toHaveLength(0); + expect(newDefinitions).toHaveLength(0); + expect(firstRow.content).toBe("# Synthetic heading 0"); + } + }); + + it("applies the declared 100-row by 100-property ceiling in bounded batches", async () => { + const seed = await fixture(100); + const input: any = { + ...plan(seed), + legacyPropertyIds: [], + propertyDefinitions: Array.from({ length: 100 }, (_, index) => ({ + id: `bounded_property_${seed.key}_${index}`, + name: `Bounded property ${index}`, + type: "text", + visibility: "always_show", + })), + }; + input.rows = seed.rows.map((row: any) => ({ + ...row, + propertyValues: input.propertyDefinitions.map((definition: any) => ({ + propertyId: definition.id, + value: `${row.documentId}:${definition.id}`, + })), + })); + + const applied: any = await runWithRequestContext({ userEmail: OWNER }, () => + action.run({ phase: "apply", plan: input }), + ); + expect(applied).toMatchObject({ + state: "applied", + counts: { rows: 100, properties: 100 }, + }); + expect( + await getDb() + .select() + .from(schema.documentPropertyValues) + .where( + inArray( + schema.documentPropertyValues.propertyId, + input.propertyDefinitions.map((definition: any) => + String(definition.id), + ), + ), + ), + ).toHaveLength(10_000); + }, 60_000); + + it("rejects source-mapped legacy fields and detects property-description drift", async () => { + const db = getDb(); + const mappedSeed = await fixture(); + const sourceId = `synthetic_source_${mappedSeed.key}`; + await db.insert(schema.contentDatabaseSources).values({ + id: sourceId, + ownerEmail: OWNER, + databaseId: mappedSeed.databaseId, + sourceType: "mock-local", + sourceName: "Synthetic source", + sourceTable: "synthetic", + }); + await db.insert(schema.contentDatabaseSourceFields).values({ + id: `synthetic_source_field_${mappedSeed.key}`, + ownerEmail: OWNER, + sourceId, + propertyId: mappedSeed.definitions[1].id, + localFieldKey: "cluster", + sourceFieldKey: "cluster", + sourceFieldLabel: "Cluster", + sourceFieldType: "text", + }); + await expect( + runWithRequestContext({ userEmail: OWNER }, () => + action.run({ phase: "apply", plan: plan(mappedSeed) }), + ), + ).rejects.toThrow("mapped to a source"); + expect( + await db + .select() + .from(schema.contentDatabaseMigrationReceipts) + .where( + eq( + schema.contentDatabaseMigrationReceipts.databaseId, + mappedSeed.databaseId, + ), + ), + ).toHaveLength(0); + + const driftSeed = await fixture(); + const driftPlan = plan(driftSeed); + const applied: any = await runWithRequestContext({ userEmail: OWNER }, () => + action.run({ phase: "apply", plan: driftPlan }), + ); + await db + .update(schema.documentPropertyDefinitions) + .set({ description: "A later synthetic edit" }) + .where( + eq( + schema.documentPropertyDefinitions.id, + driftPlan.propertyDefinitions[0].id, + ), + ); + await expect( + runWithRequestContext({ userEmail: OWNER }, () => + action.run({ + phase: "verify", + databaseId: driftSeed.databaseId, + idempotencyKey: driftPlan.idempotencyKey, + expectedPostDigest: applied.postDigest, + }), + ), + ).rejects.toThrow("drifted"); + }); + + it("requires current editor access to every row before legacy cleanup", async () => { + const seed = await fixture(); + const input = plan(seed); + const stamp = now(); + await getDb() + .insert(schema.documentShares) + .values([ + { + id: `synthetic_database_admin_${seed.key}`, + resourceId: seed.databaseDocumentId, + principalType: "user", + principalId: OUTSIDER, + role: "admin", + createdBy: OWNER, + createdAt: stamp, + }, + ...seed.rows.map((row: any, index: number) => ({ + id: `synthetic_row_editor_${seed.key}_${index}`, + resourceId: row.documentId, + principalType: "user", + principalId: OUTSIDER, + role: "editor", + createdBy: OWNER, + createdAt: stamp, + })), + ]); + const applied: any = await runWithRequestContext( + { userEmail: OUTSIDER }, + () => action.run({ phase: "apply", plan: input }), + ); + await runWithRequestContext({ userEmail: OUTSIDER }, () => + action.run({ + phase: "verify", + databaseId: seed.databaseId, + idempotencyKey: input.idempotencyKey, + expectedPostDigest: applied.postDigest, + }), + ); + await getDb() + .delete(schema.documentShares) + .where( + eq(schema.documentShares.id, `synthetic_row_editor_${seed.key}_0`), + ); + + await expect( + runWithRequestContext({ userEmail: OUTSIDER }, () => + action.run({ + phase: "finalize", + databaseId: seed.databaseId, + idempotencyKey: input.idempotencyKey, + expectedPostDigest: applied.postDigest, + }), + ), + ).rejects.toThrow(); + expect( + await getDb() + .select() + .from(schema.documentPropertyDefinitions) + .where( + inArray( + schema.documentPropertyDefinitions.id, + input.legacyPropertyIds, + ), + ), + ).toHaveLength(input.legacyPropertyIds.length); + }); + + it("performs guarded rollback/finalize and refuses drift", async () => { + const db = getDb(); + const rollbackSeed = await fixture(); + const rollbackPlan: any = structuredClone(plan(rollbackSeed)); + const beforeRollback = await readFixtureState(rollbackSeed); + const applied: any = await runWithRequestContext({ userEmail: OWNER }, () => + action.run({ phase: "apply", plan: rollbackPlan }), + ); + const rolledBack: any = await runWithRequestContext( + { userEmail: OWNER }, + () => + action.run({ + phase: "rollback", + databaseId: rollbackSeed.databaseId, + idempotencyKey: rollbackPlan.idempotencyKey, + expectedPostDigest: applied.postDigest, + }), + ); + expect(rolledBack).toMatchObject({ + state: "rolled_back", + postDigest: applied.preDigest, + }); + expect(await readFixtureState(rollbackSeed)).toEqual(beforeRollback); + const rollbackStateBeforeReplay = + await readDurableMigrationState(rollbackSeed); + const rollbackReplay: any = await runWithRequestContext( + { userEmail: OWNER }, + () => + action.run({ + phase: "rollback", + databaseId: rollbackSeed.databaseId, + idempotencyKey: rollbackPlan.idempotencyKey, + expectedPostDigest: applied.postDigest, + }), + ); + expect(rollbackReplay).toMatchObject({ + state: "rolled_back", + replayed: true, + postDigest: applied.preDigest, + }); + expect(await readFixtureState(rollbackSeed)).toEqual(beforeRollback); + expect(await readDurableMigrationState(rollbackSeed)).toEqual( + rollbackStateBeforeReplay, + ); + await expect( + runWithRequestContext({ userEmail: OWNER }, () => + action.run({ + phase: "rollback", + databaseId: rollbackSeed.databaseId, + idempotencyKey: rollbackPlan.idempotencyKey, + expectedPostDigest: "wrong-synthetic-digest", + }), + ), + ).rejects.toThrow("drifted"); + expect(await readFixtureState(rollbackSeed)).toEqual(beforeRollback); + expect( + await db + .select() + .from(schema.documentPropertyDefinitions) + .where( + eq( + schema.documentPropertyDefinitions.databaseId, + rollbackSeed.databaseId, + ), + ), + ).toHaveLength(3); + const finalizeSeed = await fixture(); + const finalizePlan: any = structuredClone(plan(finalizeSeed)); + const finalizedApply: any = await runWithRequestContext( + { userEmail: OWNER }, + () => action.run({ phase: "apply", plan: finalizePlan }), + ); + await expect( + runWithRequestContext({ userEmail: OWNER }, () => + action.run({ + phase: "finalize", + databaseId: finalizeSeed.databaseId, + idempotencyKey: finalizePlan.idempotencyKey, + expectedPostDigest: finalizedApply.postDigest, + }), + ), + ).rejects.toThrow("verified"); + const verified: any = await runWithRequestContext( + { userEmail: OWNER }, + () => + action.run({ + phase: "verify", + databaseId: finalizeSeed.databaseId, + idempotencyKey: finalizePlan.idempotencyKey, + expectedPostDigest: finalizedApply.postDigest, + }), + ); + expect(verified.state).toBe("verified"); + const verifyStateBeforeReplay = + await readDurableMigrationState(finalizeSeed); + const verifyReplay: any = await runWithRequestContext( + { userEmail: OWNER }, + () => + action.run({ + phase: "verify", + databaseId: finalizeSeed.databaseId, + idempotencyKey: finalizePlan.idempotencyKey, + expectedPostDigest: finalizedApply.postDigest, + }), + ); + expect(verifyReplay).toMatchObject({ + state: "verified", + replayed: true, + verified: true, + }); + expect(await readDurableMigrationState(finalizeSeed)).toEqual( + verifyStateBeforeReplay, + ); + const finalized: any = await runWithRequestContext( + { userEmail: OWNER }, + () => + action.run({ + phase: "finalize", + databaseId: finalizeSeed.databaseId, + idempotencyKey: finalizePlan.idempotencyKey, + expectedPostDigest: verified.postDigest, + }), + ); + expect(finalized.state).toBe("finalized"); + const finalizeStateBeforeReplay = + await readDurableMigrationState(finalizeSeed); + const finalizeReplay: any = await runWithRequestContext( + { userEmail: OWNER }, + () => + action.run({ + phase: "finalize", + databaseId: finalizeSeed.databaseId, + idempotencyKey: finalizePlan.idempotencyKey, + expectedPostDigest: verified.postDigest, + }), + ); + expect(finalizeReplay).toMatchObject({ + state: "finalized", + replayed: true, + verified: true, + }); + expect(await readDurableMigrationState(finalizeSeed)).toEqual( + finalizeStateBeforeReplay, + ); + expect( + await db + .select() + .from(schema.documentPropertyDefinitions) + .where( + and( + eq( + schema.documentPropertyDefinitions.databaseId, + finalizeSeed.databaseId, + ), + inArray( + schema.documentPropertyDefinitions.id, + finalizePlan.legacyPropertyIds, + ), + ), + ), + ).toHaveLength(0); + expect( + await db + .select() + .from(schema.documentPropertyValues) + .where( + inArray( + schema.documentPropertyValues.propertyId, + finalizePlan.legacyPropertyIds, + ), + ), + ).toHaveLength(0); + const driftSeed = await fixture(); + const driftPlan: any = structuredClone(plan(driftSeed)); + const drifted: any = await runWithRequestContext({ userEmail: OWNER }, () => + action.run({ phase: "apply", plan: driftPlan }), + ); + await db + .update(schema.documents) + .set({ content: "# Drifted synthetic body" }) + .where(eq(schema.documents.id, driftSeed.rows[0].documentId)); + await expect( + runWithRequestContext({ userEmail: OWNER }, () => + action.run({ + phase: "rollback", + databaseId: driftSeed.databaseId, + idempotencyKey: driftPlan.idempotencyKey, + expectedPostDigest: drifted.postDigest, + }), + ), + ).rejects.toThrow("drifted"); + }); + + it("allows only one competing terminal transition from a verified receipt", async () => { + const seed = await fixture(); + const input = plan(seed); + const applied: any = await runWithRequestContext({ userEmail: OWNER }, () => + action.run({ phase: "apply", plan: input }), + ); + await runWithRequestContext({ userEmail: OWNER }, () => + action.run({ + phase: "verify", + databaseId: seed.databaseId, + idempotencyKey: input.idempotencyKey, + expectedPostDigest: applied.postDigest, + }), + ); + + const transitions = await Promise.allSettled([ + runWithRequestContext({ userEmail: OWNER }, () => + action.run({ + phase: "rollback", + databaseId: seed.databaseId, + idempotencyKey: input.idempotencyKey, + expectedPostDigest: applied.postDigest, + }), + ), + runWithRequestContext({ userEmail: OWNER }, () => + action.run({ + phase: "finalize", + databaseId: seed.databaseId, + idempotencyKey: input.idempotencyKey, + expectedPostDigest: applied.postDigest, + }), + ), + ]); + + expect( + transitions.filter((transition) => transition.status === "fulfilled"), + ).toHaveLength(1); + expect( + transitions.filter((transition) => transition.status === "rejected"), + ).toHaveLength(1); + const [receipt] = await getDb() + .select() + .from(schema.contentDatabaseMigrationReceipts) + .where( + eq(schema.contentDatabaseMigrationReceipts.databaseId, seed.databaseId), + ); + expect(["rolled_back", "finalized"]).toContain(receipt.state); + }); +}); diff --git a/templates/content/actions/migrate-content-database-rows.postgres.integration.test.ts b/templates/content/actions/migrate-content-database-rows.postgres.integration.test.ts new file mode 100644 index 0000000000..ed0fd864f0 --- /dev/null +++ b/templates/content/actions/migrate-content-database-rows.postgres.integration.test.ts @@ -0,0 +1,1027 @@ +import { runWithRequestContext } from "@agent-native/core/server"; +import { eq, inArray, sql } from "drizzle-orm"; +import { + afterAll, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, +} from "vitest"; + +const flushOpenDocumentEditorToSql = vi.hoisted(() => vi.fn()); + +vi.mock("./_document-flush.js", () => ({ + flushOpenDocumentEditorToSql, +})); + +const POSTGRES_URL = process.env.CONTENT_MIGRATION_POSTGRES_URL; +const OWNER = "synthetic-postgres-migration-owner@example.test"; + +let getDb: () => any; +let schema: typeof import("../server/db/schema.js"); +let action: typeof import("./migrate-content-database-rows.js").default; +let setDocumentProperty: typeof import("./set-document-property.js").default; +let configureDocumentProperty: typeof import("./configure-document-property.js").default; +let lockContentDatabaseMutation: typeof import("./_content-database-mutation-lock.js").lockContentDatabaseMutation; +let deleteDocument: typeof import("./delete-document.js").default; +let deleteContentDatabase: typeof import("./delete-content-database.js").default; +let restoreDocument: typeof import("./restore-document.js").default; +let restoreContentDatabase: typeof import("./restore-content-database.js").default; +let permanentlyDeleteDocument: typeof import("./permanently-delete-document.js").default; + +beforeAll(async () => { + if (!POSTGRES_URL) return; + const databaseName = new URL(POSTGRES_URL).pathname.slice(1).toLowerCase(); + if (!databaseName.includes("test")) { + throw new Error( + "CONTENT_MIGRATION_POSTGRES_URL must name an isolated test database.", + ); + } + process.env.DATABASE_URL = POSTGRES_URL; + const database = await import("../server/db/index.js"); + getDb = database.getDb; + schema = database.schema; + action = (await import("./migrate-content-database-rows.js")).default; + setDocumentProperty = (await import("./set-document-property.js")).default; + configureDocumentProperty = (await import("./configure-document-property.js")) + .default; + lockContentDatabaseMutation = ( + await import("./_content-database-mutation-lock.js") + ).lockContentDatabaseMutation; + deleteDocument = (await import("./delete-document.js")).default; + deleteContentDatabase = (await import("./delete-content-database.js")) + .default; + restoreDocument = (await import("./restore-document.js")).default; + restoreContentDatabase = (await import("./restore-content-database.js")) + .default; + permanentlyDeleteDocument = (await import("./permanently-delete-document.js")) + .default; + await (await import("../server/plugins/db.js")).default(undefined as any); +}, 60_000); + +beforeEach(() => { + flushOpenDocumentEditorToSql.mockReset(); + flushOpenDocumentEditorToSql.mockResolvedValue(undefined); +}); + +afterAll(() => { + delete process.env.DATABASE_URL; +}); + +async function fixture() { + const db = getDb(); + const key = `${Date.now()}_${Math.random().toString(36).slice(2)}`; + const stamp = "2026-01-01T00:00:00.000Z"; + const databaseId = `postgres_migration_db_${key}`; + const databaseDocumentId = `postgres_migration_page_${key}`; + const documentId = `postgres_migration_row_${key}`; + const itemId = `postgres_migration_item_${key}`; + const protectedPropertyId = `status_${key}`; + const legacyPropertyId = `legacy_${key}`; + const newPropertyId = `reported_by_${key}`; + await db.insert(schema.documents).values([ + { + id: databaseDocumentId, + ownerEmail: OWNER, + spaceId: "synthetic_space", + title: "Synthetic Postgres migration database", + content: "", + visibility: "private", + createdAt: stamp, + updatedAt: stamp, + }, + { + id: documentId, + ownerEmail: OWNER, + spaceId: "synthetic_space", + parentId: databaseDocumentId, + title: "Synthetic row", + content: "# Before", + visibility: "private", + hideFromSearch: 1, + createdAt: stamp, + updatedAt: stamp, + }, + ]); + await db.insert(schema.contentDatabases).values({ + id: databaseId, + ownerEmail: OWNER, + spaceId: "synthetic_space", + documentId: databaseDocumentId, + title: "Synthetic Postgres migration database", + createdAt: stamp, + updatedAt: stamp, + }); + await db.insert(schema.contentDatabaseItems).values({ + id: itemId, + ownerEmail: OWNER, + databaseId, + documentId, + position: 0, + createdAt: stamp, + updatedAt: stamp, + }); + await db.insert(schema.documentPropertyDefinitions).values([ + { + id: protectedPropertyId, + ownerEmail: OWNER, + databaseId, + name: "Status", + type: "status", + visibility: "always_show", + optionsJson: "{}", + position: 0, + createdAt: stamp, + updatedAt: stamp, + }, + { + id: legacyPropertyId, + ownerEmail: OWNER, + databaseId, + name: "Legacy", + type: "text", + visibility: "always_show", + optionsJson: "{}", + position: 1, + createdAt: stamp, + updatedAt: stamp, + }, + ]); + await db.insert(schema.documentPropertyValues).values({ + id: `postgres_migration_status_${key}`, + ownerEmail: OWNER, + documentId, + propertyId: protectedPropertyId, + valueJson: '"open"', + createdAt: stamp, + updatedAt: stamp, + }); + return { + databaseId, + databaseDocumentId, + documentId, + newPropertyId, + protectedPropertyId, + plan: { + databaseId, + databaseDocumentId, + idempotencyKey: `postgres-key-${key}`, + expectedRowCount: 1, + legacyPropertyIds: [legacyPropertyId], + propertyDefinitions: [ + { + id: newPropertyId, + name: "Reported by", + type: "text", + visibility: "always_show", + }, + ], + rows: [ + { + itemId, + documentId, + expectedUpdatedAt: stamp, + content: "# Migrated", + propertyValues: [{ propertyId: newPropertyId, value: "Synthetic" }], + protectedPropertyValues: [ + { propertyId: protectedPropertyId, valueJson: '"open"' }, + ], + }, + ], + }, + }; +} + +async function cleanupFixture(seed: Awaited>) { + await getDb().transaction(async (tx: any) => { + await tx + .delete(schema.contentDatabaseMigrationReceipts) + .where( + eq(schema.contentDatabaseMigrationReceipts.databaseId, seed.databaseId), + ); + await tx + .delete(schema.documentVersions) + .where(eq(schema.documentVersions.documentId, seed.documentId)); + await tx + .delete(schema.documentPropertyValues) + .where(eq(schema.documentPropertyValues.documentId, seed.documentId)); + await tx + .delete(schema.documentPropertyDefinitions) + .where( + eq(schema.documentPropertyDefinitions.databaseId, seed.databaseId), + ); + await tx + .delete(schema.contentDatabaseItems) + .where(eq(schema.contentDatabaseItems.databaseId, seed.databaseId)); + await tx + .delete(schema.contentDatabases) + .where(eq(schema.contentDatabases.id, seed.databaseId)); + await tx + .delete(schema.documents) + .where( + inArray(schema.documents.id, [ + seed.documentId, + seed.databaseDocumentId, + ]), + ); + }); +} + +async function waitForPostgresLockWait(minimum: number) { + for (let attempt = 0; attempt < 100; attempt += 1) { + const result: any = await getDb().execute( + sql.raw( + "SELECT count(*)::int AS waiting FROM pg_locks WHERE NOT granted AND locktype IN ('advisory', 'transactionid')", + ), + ); + const rows = Array.isArray(result) ? result : (result.rows ?? []); + if (Number(rows[0]?.waiting) >= minimum) return; + await new Promise((resolve) => setTimeout(resolve, 10)); + } + throw new Error("Migration did not enter the expected PostgreSQL lock wait."); +} + +const postgresSuite = POSTGRES_URL ? describe : describe.skip; + +postgresSuite("migrate-content-database-rows PostgreSQL locking", () => { + it("rejects stale plans before requesting an editor flush", async () => { + const seed = await fixture(); + try { + seed.plan.rows[0]!.expectedUpdatedAt = "2025-12-31T00:00:00.000Z"; + await expect( + runWithRequestContext({ userEmail: OWNER }, () => + action.run({ phase: "apply", plan: seed.plan }), + ), + ).rejects.toThrow("Stale row"); + expect(flushOpenDocumentEditorToSql).not.toHaveBeenCalled(); + } finally { + await cleanupFixture(seed); + } + }); + + it("revalidates an editor save before applying the migration", async () => { + const seed = await fixture(); + try { + flushOpenDocumentEditorToSql.mockImplementationOnce( + async (args: { documentId: string }) => { + await getDb() + .update(schema.documents) + .set({ + content: "# New live editor body", + updatedAt: "2026-01-01T00:00:01.000Z", + }) + .where(eq(schema.documents.id, args.documentId)); + }, + ); + await expect( + runWithRequestContext({ userEmail: OWNER }, () => + action.run({ phase: "apply", plan: seed.plan }), + ), + ).rejects.toThrow("Stale row"); + expect( + await getDb() + .select() + .from(schema.contentDatabaseMigrationReceipts) + .where( + eq( + schema.contentDatabaseMigrationReceipts.databaseId, + seed.databaseId, + ), + ), + ).toHaveLength(0); + } finally { + await cleanupFixture(seed); + } + }); + + it.each(["apply", "rollback"] as const)( + "holds the durable database lock while flushing live editors during %s", + async (phase) => { + const seed = await fixture(); + let releaseFlush = () => {}; + let contender: Promise | undefined; + let operation: Promise | undefined; + try { + const applied = + phase === "rollback" + ? await runWithRequestContext({ userEmail: OWNER }, () => + action.run({ phase: "apply", plan: seed.plan }), + ) + : null; + const flushReleased = new Promise((resolve) => { + releaseFlush = resolve; + }); + let flushEntered!: () => void; + const flushStarted = new Promise((resolve) => { + flushEntered = resolve; + }); + flushOpenDocumentEditorToSql.mockImplementationOnce( + async (args: { documentId: string }) => { + await getDb() + .update(schema.documents) + .set({ content: phase === "apply" ? "# Before" : "# Migrated" }) + .where(eq(schema.documents.id, args.documentId)); + flushEntered(); + await flushReleased; + }, + ); + + operation = runWithRequestContext({ userEmail: OWNER }, () => + phase === "apply" + ? action.run({ phase: "apply", plan: seed.plan }) + : action.run({ + phase: "rollback", + databaseId: seed.databaseId, + idempotencyKey: seed.plan.idempotencyKey, + expectedPostDigest: applied!.postDigest, + }), + ); + await flushStarted; + + let contenderEntered = false; + contender = getDb().transaction(async (tx: any) => { + await lockContentDatabaseMutation(tx, seed.databaseId); + contenderEntered = true; + }); + await waitForPostgresLockWait(1); + expect(contenderEntered).toBe(false); + + releaseFlush(); + await operation; + await contender; + expect(contenderEntered).toBe(true); + } finally { + releaseFlush(); + await Promise.allSettled( + [operation, contender].filter( + (pending): pending is Promise => Boolean(pending), + ), + ); + await cleanupFixture(seed); + } + }, + 60_000, + ); + + it("serializes restoring a trashed row behind the migration snapshot", async () => { + const seed = await fixture(); + const stamp = "2026-01-01T00:00:00.000Z"; + const restoredDocumentId = `postgres_restored_row_${seed.databaseId}`; + const restoredItemId = `postgres_restored_item_${seed.databaseId}`; + await getDb().insert(schema.documents).values({ + id: restoredDocumentId, + ownerEmail: OWNER, + spaceId: "synthetic_space", + parentId: seed.databaseDocumentId, + title: "Synthetic trashed row", + content: "# Not migrated", + visibility: "private", + hideFromSearch: 1, + trashedAt: stamp, + trashRootId: restoredDocumentId, + createdAt: stamp, + updatedAt: stamp, + }); + await getDb().insert(schema.contentDatabaseItems).values({ + id: restoredItemId, + ownerEmail: OWNER, + databaseId: seed.databaseId, + documentId: restoredDocumentId, + position: 1, + createdAt: stamp, + updatedAt: stamp, + }); + + let releaseFlush = () => {}; + let migration: Promise | undefined; + let restore: Promise | undefined; + try { + const flushReleased = new Promise((resolve) => { + releaseFlush = resolve; + }); + let flushEntered!: () => void; + const flushStarted = new Promise((resolve) => { + flushEntered = resolve; + }); + flushOpenDocumentEditorToSql.mockImplementationOnce(async () => { + flushEntered(); + await flushReleased; + }); + + migration = runWithRequestContext({ userEmail: OWNER }, () => + action.run({ phase: "apply", plan: seed.plan }), + ); + await flushStarted; + restore = runWithRequestContext({ userEmail: OWNER }, () => + restoreDocument.run({ id: restoredDocumentId }), + ); + await waitForPostgresLockWait(1); + expect( + ( + await getDb() + .select({ trashedAt: schema.documents.trashedAt }) + .from(schema.documents) + .where(eq(schema.documents.id, restoredDocumentId)) + )[0]?.trashedAt, + ).toBe(stamp); + + releaseFlush(); + const applied = await migration; + await restore; + await expect( + runWithRequestContext({ userEmail: OWNER }, () => + action.run({ + phase: "verify", + databaseId: seed.databaseId, + idempotencyKey: seed.plan.idempotencyKey, + expectedPostDigest: applied.postDigest, + }), + ), + ).rejects.toThrow("Migration has drifted; verification is refused."); + } finally { + releaseFlush(); + await Promise.allSettled( + [migration, restore].filter((pending): pending is Promise => + Boolean(pending), + ), + ); + await cleanupFixture(seed); + await getDb() + .delete(schema.documents) + .where(eq(schema.documents.id, restoredDocumentId)); + } + }, 60_000); + + it.each(["same-trash-root", "active"] as const)( + "rebuilds permanent-delete scope after a concurrent %s row insertion", + async (rowState) => { + const seed = await fixture(); + const stamp = "2026-01-01T00:00:00.000Z"; + const extraDocumentId = `postgres_delete_row_${seed.databaseId}`; + const extraItemId = `postgres_delete_item_${seed.databaseId}`; + await runWithRequestContext({ userEmail: OWNER }, () => + deleteContentDatabase.run({ databaseId: seed.databaseId }), + ); + + let requestInsertion = () => {}; + let releaseHolder = () => {}; + let holder: Promise | undefined; + let deletion: Promise | undefined; + try { + const insertionRequested = new Promise((resolve) => { + requestInsertion = resolve; + }); + const holderReleased = new Promise((resolve) => { + releaseHolder = resolve; + }); + let holderEntered!: () => void; + const holderStarted = new Promise((resolve) => { + holderEntered = resolve; + }); + let rowInserted!: () => void; + const insertionCompleted = new Promise((resolve) => { + rowInserted = resolve; + }); + holder = getDb().transaction(async (tx: any) => { + await lockContentDatabaseMutation(tx, seed.databaseId); + holderEntered(); + await insertionRequested; + await tx.insert(schema.documents).values({ + id: extraDocumentId, + ownerEmail: OWNER, + spaceId: "synthetic_space", + parentId: seed.databaseDocumentId, + title: "Concurrent synthetic row", + content: "# Concurrent", + visibility: "private", + hideFromSearch: 1, + trashedAt: rowState === "same-trash-root" ? stamp : null, + trashRootId: + rowState === "same-trash-root" ? seed.databaseDocumentId : null, + createdAt: stamp, + updatedAt: stamp, + }); + await tx.insert(schema.contentDatabaseItems).values({ + id: extraItemId, + ownerEmail: OWNER, + databaseId: seed.databaseId, + documentId: extraDocumentId, + position: 1, + createdAt: stamp, + updatedAt: stamp, + }); + rowInserted(); + await holderReleased; + }); + await holderStarted; + + deletion = runWithRequestContext({ userEmail: OWNER }, () => + permanentlyDeleteDocument.run({ id: seed.databaseDocumentId }), + ); + const deletionExpectation = + rowState === "active" + ? expect(deletion).rejects.toThrow( + "Database contains an active row outside this Trash item", + ) + : null; + await waitForPostgresLockWait(1); + requestInsertion(); + await insertionCompleted; + releaseHolder(); + await holder; + + if (deletionExpectation) { + await deletionExpectation; + expect( + await getDb() + .select() + .from(schema.contentDatabases) + .where(eq(schema.contentDatabases.id, seed.databaseId)), + ).toHaveLength(1); + expect( + await getDb() + .select() + .from(schema.documents) + .where(eq(schema.documents.id, extraDocumentId)), + ).toHaveLength(1); + } else { + await deletion; + expect( + await getDb() + .select() + .from(schema.contentDatabaseItems) + .where(eq(schema.contentDatabaseItems.id, extraItemId)), + ).toHaveLength(0); + expect( + await getDb() + .select() + .from(schema.documents) + .where(eq(schema.documents.id, extraDocumentId)), + ).toHaveLength(0); + } + } finally { + requestInsertion(); + releaseHolder(); + await Promise.allSettled( + [holder, deletion].filter((pending): pending is Promise => + Boolean(pending), + ), + ); + await cleanupFixture(seed); + await getDb() + .delete(schema.documents) + .where(eq(schema.documents.id, extraDocumentId)); + } + }, + 60_000, + ); + + it("retries permanent deletion when a new external membership expands the lock set", async () => { + const seed = await fixture(); + const stamp = "2026-01-01T00:00:00.000Z"; + const externalDatabaseId = `aaa_external_${seed.databaseId}`; + const externalDatabaseDocumentId = `external_page_${seed.databaseId}`; + const externalItemId = `external_item_${seed.databaseId}`; + await getDb().insert(schema.documents).values({ + id: externalDatabaseDocumentId, + ownerEmail: OWNER, + spaceId: "synthetic_space", + title: "External synthetic database", + content: "", + visibility: "private", + createdAt: stamp, + updatedAt: stamp, + }); + await getDb().insert(schema.contentDatabases).values({ + id: externalDatabaseId, + ownerEmail: OWNER, + spaceId: "synthetic_space", + documentId: externalDatabaseDocumentId, + title: "External synthetic database", + createdAt: stamp, + updatedAt: stamp, + }); + await runWithRequestContext({ userEmail: OWNER }, () => + deleteDocument.run({ id: seed.documentId }), + ); + + let releaseHolder = () => {}; + let holder: Promise | undefined; + let deletion: Promise | undefined; + try { + const holderReleased = new Promise((resolve) => { + releaseHolder = resolve; + }); + let holderEntered!: () => void; + const holderStarted = new Promise((resolve) => { + holderEntered = resolve; + }); + holder = getDb().transaction(async (tx: any) => { + await lockContentDatabaseMutation(tx, seed.databaseId); + holderEntered(); + await holderReleased; + }); + await holderStarted; + + deletion = runWithRequestContext({ userEmail: OWNER }, () => + permanentlyDeleteDocument.run({ id: seed.documentId }), + ); + await waitForPostgresLockWait(1); + await getDb().transaction(async (tx: any) => { + await lockContentDatabaseMutation(tx, externalDatabaseId); + await tx.insert(schema.contentDatabaseItems).values({ + id: externalItemId, + ownerEmail: OWNER, + databaseId: externalDatabaseId, + documentId: seed.documentId, + position: 0, + createdAt: stamp, + updatedAt: stamp, + }); + }); + releaseHolder(); + await holder; + await deletion; + + expect( + await getDb() + .select() + .from(schema.documents) + .where(eq(schema.documents.id, seed.documentId)), + ).toHaveLength(0); + expect( + await getDb() + .select() + .from(schema.contentDatabaseItems) + .where(eq(schema.contentDatabaseItems.id, externalItemId)), + ).toHaveLength(0); + expect( + await getDb() + .select() + .from(schema.contentDatabases) + .where(eq(schema.contentDatabases.id, externalDatabaseId)), + ).toHaveLength(1); + } finally { + releaseHolder(); + await Promise.allSettled( + [holder, deletion].filter((pending): pending is Promise => + Boolean(pending), + ), + ); + await cleanupFixture(seed); + await getDb() + .delete(schema.contentDatabaseItems) + .where(eq(schema.contentDatabaseItems.databaseId, externalDatabaseId)); + await getDb() + .delete(schema.contentDatabases) + .where(eq(schema.contentDatabases.id, externalDatabaseId)); + await getDb() + .delete(schema.documents) + .where(eq(schema.documents.id, externalDatabaseDocumentId)); + } + }, 60_000); + + it("refuses permanent deletion when restore wins the lifecycle locks", async () => { + const seed = await fixture(); + await runWithRequestContext({ userEmail: OWNER }, () => + deleteContentDatabase.run({ databaseId: seed.databaseId }), + ); + let releaseHolder = () => {}; + let holder: Promise | undefined; + let restore: Promise | undefined; + let deletion: Promise | undefined; + try { + const holderReleased = new Promise((resolve) => { + releaseHolder = resolve; + }); + let holderEntered!: () => void; + const holderStarted = new Promise((resolve) => { + holderEntered = resolve; + }); + holder = getDb().transaction(async (tx: any) => { + await lockContentDatabaseMutation(tx, seed.databaseId); + holderEntered(); + await holderReleased; + }); + await holderStarted; + + restore = runWithRequestContext({ userEmail: OWNER }, () => + restoreContentDatabase.run({ databaseId: seed.databaseId }), + ); + await waitForPostgresLockWait(1); + deletion = runWithRequestContext({ userEmail: OWNER }, () => + permanentlyDeleteDocument.run({ id: seed.databaseDocumentId }), + ); + await new Promise((resolve) => setTimeout(resolve, 100)); + releaseHolder(); + await holder; + await restore; + await expect(deletion).rejects.toThrow( + "Document must be in Trash and be a Trash root before permanent deletion", + ); + + expect( + ( + await getDb() + .select({ deletedAt: schema.contentDatabases.deletedAt }) + .from(schema.contentDatabases) + .where(eq(schema.contentDatabases.id, seed.databaseId)) + )[0]?.deletedAt, + ).toBeNull(); + expect( + ( + await getDb() + .select({ trashedAt: schema.documents.trashedAt }) + .from(schema.documents) + .where(eq(schema.documents.id, seed.databaseDocumentId)) + )[0]?.trashedAt, + ).toBeNull(); + } finally { + releaseHolder(); + await Promise.allSettled( + [holder, restore, deletion].filter( + (pending): pending is Promise => Boolean(pending), + ), + ); + await cleanupFixture(seed); + } + }, 60_000); + + it("removes a receipt when permanent deletion follows a migration that held the database lock", async () => { + const seed = await fixture(); + const rootId = `postgres_migration_root_${seed.databaseId}`; + await getDb().insert(schema.documents).values({ + id: rootId, + ownerEmail: OWNER, + spaceId: "synthetic_space", + title: "Synthetic parent", + content: "", + visibility: "private", + createdAt: "2026-01-01T00:00:00.000Z", + updatedAt: "2026-01-01T00:00:00.000Z", + }); + await getDb() + .update(schema.documents) + .set({ parentId: rootId }) + .where(eq(schema.documents.id, seed.databaseDocumentId)); + let releaseFlush = () => {}; + let migration: Promise | undefined; + let trash: Promise | undefined; + try { + const flushReleased = new Promise((resolve) => { + releaseFlush = resolve; + }); + let flushEntered = () => {}; + const flushStarted = new Promise((resolve) => { + flushEntered = resolve; + }); + flushOpenDocumentEditorToSql.mockImplementationOnce(async () => { + flushEntered(); + await flushReleased; + }); + + migration = runWithRequestContext({ userEmail: OWNER }, () => + action.run({ phase: "apply", plan: seed.plan }), + ); + await flushStarted; + trash = runWithRequestContext({ userEmail: OWNER }, () => + deleteDocument.run({ id: rootId }), + ); + await waitForPostgresLockWait(1); + + releaseFlush(); + await migration; + await trash; + await runWithRequestContext({ userEmail: OWNER }, () => + permanentlyDeleteDocument.run({ id: rootId }), + ); + + expect( + await getDb() + .select() + .from(schema.contentDatabaseMigrationReceipts) + .where( + eq( + schema.contentDatabaseMigrationReceipts.databaseId, + seed.databaseId, + ), + ), + ).toHaveLength(0); + expect( + await getDb() + .select() + .from(schema.contentDatabases) + .where(eq(schema.contentDatabases.id, seed.databaseId)), + ).toHaveLength(0); + expect( + await getDb() + .select() + .from(schema.contentDatabaseItems) + .where(eq(schema.contentDatabaseItems.databaseId, seed.databaseId)), + ).toHaveLength(0); + } finally { + releaseFlush(); + await Promise.allSettled( + [migration, trash].filter((pending): pending is Promise => + Boolean(pending), + ), + ); + await cleanupFixture(seed); + await getDb() + .delete(schema.documents) + .where(eq(schema.documents.id, rootId)); + } + }, 60_000); + + it("fails a migration cleanly when database deletion wins the durable lock", async () => { + const seed = await fixture(); + const gate = 64_058; + const trigger = `synthetic_delete_gate_${seed.databaseId}`; + const functionName = `synthetic_delete_gate_fn_${seed.databaseId}`; + await getDb().execute( + sql.raw( + `CREATE FUNCTION ${functionName}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN PERFORM pg_advisory_xact_lock(${gate}); RETURN NEW; END; $$`, + ), + ); + await getDb().execute( + sql.raw( + `CREATE TRIGGER ${trigger} BEFORE UPDATE ON content_databases FOR EACH ROW WHEN (OLD.deleted_at IS NULL AND NEW.deleted_at IS NOT NULL AND NEW.owner_email = '${OWNER}') EXECUTE FUNCTION ${functionName}()`, + ), + ); + let releaseGate = () => {}; + let gateHolder: Promise | undefined; + let trash: Promise | undefined; + let permanentDelete: Promise | undefined; + let migrationExpectation: Promise | undefined; + try { + const gateReleased = new Promise((resolve) => { + releaseGate = resolve; + }); + let gateHeld = () => {}; + const gateAcquired = new Promise((resolve) => { + gateHeld = resolve; + }); + gateHolder = getDb().transaction(async (tx: any) => { + await tx.execute(sql.raw(`SELECT pg_advisory_xact_lock(${gate})`)); + gateHeld(); + await gateReleased; + }); + await gateAcquired; + trash = runWithRequestContext({ userEmail: OWNER }, () => + deleteContentDatabase.run({ databaseId: seed.databaseId }), + ); + await waitForPostgresLockWait(1); + + const migration = runWithRequestContext({ userEmail: OWNER }, () => + action.run({ phase: "apply", plan: seed.plan }), + ); + migrationExpectation = Promise.resolve( + expect(migration).rejects.toThrow("Database not found"), + ); + await waitForPostgresLockWait(2); + releaseGate(); + await gateHolder; + await trash; + permanentDelete = runWithRequestContext({ userEmail: OWNER }, () => + permanentlyDeleteDocument.run({ id: seed.databaseDocumentId }), + ); + await Promise.all([migrationExpectation, permanentDelete]); + + expect( + await getDb() + .select() + .from(schema.contentDatabaseMigrationReceipts) + .where( + eq( + schema.contentDatabaseMigrationReceipts.databaseId, + seed.databaseId, + ), + ), + ).toHaveLength(0); + expect( + await getDb() + .select() + .from(schema.contentDatabases) + .where(eq(schema.contentDatabases.id, seed.databaseId)), + ).toHaveLength(0); + expect(flushOpenDocumentEditorToSql).not.toHaveBeenCalled(); + } finally { + releaseGate(); + await Promise.allSettled( + [gateHolder, trash, permanentDelete, migrationExpectation].filter( + (pending): pending is Promise => Boolean(pending), + ), + ); + await getDb().execute( + sql.raw(`DROP TRIGGER IF EXISTS ${trigger} ON content_databases`), + ); + await getDb().execute( + sql.raw(`DROP FUNCTION IF EXISTS ${functionName}()`), + ); + await cleanupFixture(seed); + } + }, 60_000); + + it.each(["value", "schema"] as const)( + "serializes a migration behind a real %s writer transaction", + async (writer) => { + const seed = await fixture(); + const gate = 64_057; + const trigger = `synthetic_migration_gate_${seed.databaseId}`; + const functionName = `synthetic_migration_gate_fn_${seed.databaseId}`; + const table = + writer === "value" + ? "document_property_values" + : "document_property_definitions"; + await getDb().execute( + sql.raw( + `CREATE FUNCTION ${functionName}() RETURNS trigger LANGUAGE plpgsql AS $$ BEGIN PERFORM pg_advisory_xact_lock(${gate}); RETURN NEW; END; $$`, + ), + ); + await getDb().execute( + sql.raw( + `CREATE TRIGGER ${trigger} BEFORE ${writer === "value" ? "UPDATE" : "INSERT"} ON ${table} FOR EACH ROW WHEN (NEW.owner_email = '${OWNER}') EXECUTE FUNCTION ${functionName}()`, + ), + ); + let releaseGate = () => {}; + let gateHolder: Promise | undefined; + let concurrentWriter: Promise | undefined; + let migrationExpectation: Promise | undefined; + try { + const gateReleased = new Promise((resolve) => { + releaseGate = resolve; + }); + let gateHeld!: () => void; + const gateAcquired = new Promise((resolve) => { + gateHeld = resolve; + }); + gateHolder = getDb().transaction(async (tx: any) => { + await tx.execute(sql.raw(`SELECT pg_advisory_xact_lock(${gate})`)); + gateHeld(); + await gateReleased; + }); + await gateAcquired; + concurrentWriter = runWithRequestContext({ userEmail: OWNER }, () => + writer === "value" + ? setDocumentProperty.run({ + documentId: seed.documentId, + databaseId: seed.databaseId, + propertyId: seed.protectedPropertyId, + value: "closed", + }) + : configureDocumentProperty.run({ + documentId: seed.documentId, + databaseId: seed.databaseId, + name: "Reported by", + type: "text", + visibility: "always_show", + }), + ); + await waitForPostgresLockWait(1); + const migration = runWithRequestContext({ userEmail: OWNER }, () => + action.run({ phase: "apply", plan: seed.plan }), + ); + migrationExpectation = Promise.resolve( + expect(migration).rejects.toThrow( + writer === "value" + ? "Protected property values no longer match persisted values" + : "New property definition collides with an existing definition", + ), + ); + await waitForPostgresLockWait(2); + releaseGate(); + await gateHolder; + await concurrentWriter; + await migrationExpectation; + expect( + await getDb() + .select() + .from(schema.contentDatabaseMigrationReceipts) + .where( + eq( + schema.contentDatabaseMigrationReceipts.databaseId, + seed.databaseId, + ), + ), + ).toHaveLength(0); + } finally { + releaseGate(); + await Promise.allSettled( + [gateHolder, concurrentWriter, migrationExpectation].filter( + (pending): pending is Promise => Boolean(pending), + ), + ); + await getDb().execute( + sql.raw(`DROP TRIGGER IF EXISTS ${trigger} ON ${table}`), + ); + await getDb().execute( + sql.raw(`DROP FUNCTION IF EXISTS ${functionName}()`), + ); + await cleanupFixture(seed); + } + }, + 60_000, + ); +}); diff --git a/templates/content/actions/migrate-content-database-rows.ts b/templates/content/actions/migrate-content-database-rows.ts new file mode 100644 index 0000000000..0abb8c35ee --- /dev/null +++ b/templates/content/actions/migrate-content-database-rows.ts @@ -0,0 +1,830 @@ +import { defineAction } from "@agent-native/core"; +import { writeAppState } from "@agent-native/core/application-state"; +import { isLocalDatabase, isPostgres } from "@agent-native/core/db"; +import { assertAccess } from "@agent-native/core/sharing"; +import { and, eq, inArray } from "drizzle-orm"; +import { z } from "zod"; + +import { getDb, schema } from "../server/db/index.js"; +import { + lockContentDatabaseMutation, + touchContentDatabase, + withContentDatabaseMutationLock, +} from "./_content-database-mutation-lock.js"; +import { + applyMigration, + deterministicId, + digest, + migrationPlanSchema, + snapshotDigest, + snapshotMigration, + serializeMigrationValue, + validatePlan, +} from "./_content-database-row-migration.js"; +import { lockDatabaseMemberships } from "./_database-membership-lock.js"; +import { flushOpenDocumentEditorToSql } from "./_document-flush.js"; + +const operationalSchema = z.discriminatedUnion("phase", [ + z.object({ phase: z.literal("validate"), plan: migrationPlanSchema }), + z.object({ phase: z.literal("apply"), plan: migrationPlanSchema }), + z.object({ + phase: z.literal("verify"), + databaseId: z.string().min(1), + idempotencyKey: z.string().min(1), + expectedPostDigest: z.string().min(1), + }), + z.object({ + phase: z.literal("rollback"), + databaseId: z.string().min(1), + idempotencyKey: z.string().min(1), + expectedPostDigest: z.string().min(1), + }), + z.object({ + phase: z.literal("finalize"), + databaseId: z.string().min(1), + idempotencyKey: z.string().min(1), + expectedPostDigest: z.string().min(1), + }), +]); + +function parseJson(text: string) { + try { + return JSON.parse(text) as any; + } catch { + throw new Error("Migration receipt is corrupt."); + } +} +function receiptResult(receipt: any, replayed: boolean) { + const { + plan: _plan, + transitionExpectedPostDigest: _transitionExpectedPostDigest, + ...result + } = parseJson(receipt.resultJson); + return { + ...result, + receiptId: receipt.id, + state: receipt.state, + preDigest: receipt.preDigest, + postDigest: receipt.postDigest, + replayed, + verified: result.verified === true, + }; +} + +async function lockCurrentDatabaseMemberships(tx: any, databaseId: string) { + const memberships = await tx + .select({ id: schema.contentDatabaseItems.id }) + .from(schema.contentDatabaseItems) + .where(eq(schema.contentDatabaseItems.databaseId, databaseId)); + await lockDatabaseMemberships( + tx, + memberships.map((membership: { id: string }) => membership.id), + ); +} + +async function flushMigrationDocuments(rows: Array<{ documentId: string }>) { + const accesses = await Promise.all( + rows.map((row) => assertAccess("document", row.documentId, "editor")), + ); + const flushes = await Promise.allSettled( + rows.map((row, index) => + flushOpenDocumentEditorToSql({ + documentId: row.documentId, + ownerEmail: accesses[index]?.resource.ownerEmail, + }), + ), + ); + const failed = flushes.find( + (flush): flush is PromiseRejectedResult => flush.status === "rejected", + ); + if (failed) throw failed.reason; +} + +export default defineAction({ + description: + "Atomically migrate every active row in one ordinary Content database without attached Sources: validates an exact bounded plan, snapshots bodies, writes only new safe properties, and supports guarded rollback or legacy-property finalization.", + schema: operationalSchema, + audit: { + recordInputs: false, + target: (args) => ({ + type: "content-database", + id: + args.phase === "apply" || args.phase === "validate" + ? args.plan.databaseId + : args.databaseId, + visibility: "private", + }), + }, + needsApproval: (args) => + args.phase === "rollback" || args.phase === "finalize", + run: async (args) => { + const db = getDb(); + const databaseId = + args.phase === "apply" || args.phase === "validate" + ? args.plan.databaseId + : args.databaseId; + const localDatabase = isLocalDatabase(); + const flushUnderDurableLock = isPostgres() && !localDatabase; + return withContentDatabaseMutationLock(databaseId, async () => { + const [database] = await db + .select() + .from(schema.contentDatabases) + .where(eq(schema.contentDatabases.id, databaseId)); + if (!database) throw new Error("Database not found."); + await assertAccess("document", database.documentId, "admin"); + if (args.phase === "apply") { + const replay = await db.transaction(async (tx) => { + const [existing] = await tx + .select() + .from(schema.contentDatabaseMigrationReceipts) + .where( + and( + eq( + schema.contentDatabaseMigrationReceipts.databaseId, + args.plan.databaseId, + ), + eq( + schema.contentDatabaseMigrationReceipts.idempotencyKey, + args.plan.idempotencyKey, + ), + ), + ); + const planHash = digest(args.plan); + if (existing) { + if (existing.planHash !== planHash) + throw new Error( + "Idempotency key was already used with a different migration plan.", + ); + if (existing.state !== "applied" && existing.state !== "verified") + throw new Error( + `Migration receipt is already ${existing.state}.`, + ); + if ( + snapshotDigest( + await snapshotMigration(tx, args.plan.databaseId), + ) !== existing.postDigest + ) + throw new Error( + "Applied migration has drifted; replay is refused.", + ); + return receiptResult(existing, true); + } + validatePlan( + args.plan, + await snapshotMigration(tx, args.plan.databaseId), + ); + return null; + }); + if (replay) return replay; + if (!localDatabase && !flushUnderDurableLock) { + throw new Error( + "Database row migration requires PostgreSQL or a local SQLite/PGlite database so live editor saves can be serialized safely.", + ); + } + if (!flushUnderDurableLock) { + await flushMigrationDocuments(args.plan.rows); + } + } + if (args.phase === "rollback") { + const preflight = await db.transaction(async (tx) => { + const [receipt] = await tx + .select() + .from(schema.contentDatabaseMigrationReceipts) + .where( + and( + eq( + schema.contentDatabaseMigrationReceipts.databaseId, + args.databaseId, + ), + eq( + schema.contentDatabaseMigrationReceipts.idempotencyKey, + args.idempotencyKey, + ), + ), + ); + if (!receipt) throw new Error("Migration receipt not found."); + const result = parseJson(receipt.resultJson); + const current = await snapshotMigration(tx, args.databaseId); + if (receipt.state === "rolled_back") { + if ( + result.transitionExpectedPostDigest !== args.expectedPostDigest || + snapshotDigest(current) !== receipt.postDigest + ) + throw new Error( + "Terminal migration result has drifted; replay is refused.", + ); + return { replay: receiptResult(receipt, true), versions: [] }; + } + if (receipt.state !== "applied" && receipt.state !== "verified") + throw new Error(`Migration receipt is already ${receipt.state}.`); + if ( + receipt.postDigest !== args.expectedPostDigest || + snapshotDigest(current) !== receipt.postDigest + ) + throw new Error( + "Migration has drifted; guarded operation is refused.", + ); + return { + replay: null, + versions: parseJson(receipt.rollbackJson).versions ?? [], + }; + }); + if (preflight.replay) return preflight.replay; + if (!localDatabase && !flushUnderDurableLock) { + throw new Error( + "Database row migration requires PostgreSQL or a local SQLite/PGlite database so live editor saves can be serialized safely.", + ); + } + if (!flushUnderDurableLock) { + await flushMigrationDocuments(preflight.versions); + } + } + if (args.phase === "verify" || args.phase === "finalize") { + const [receipt] = await db + .select() + .from(schema.contentDatabaseMigrationReceipts) + .where( + and( + eq( + schema.contentDatabaseMigrationReceipts.databaseId, + args.databaseId, + ), + eq( + schema.contentDatabaseMigrationReceipts.idempotencyKey, + args.idempotencyKey, + ), + ), + ); + if (!receipt) throw new Error("Migration receipt not found."); + if (receipt.state === "applied" || receipt.state === "verified") { + const plan = parseJson(receipt.resultJson).plan; + if (!plan) + throw new Error("Migration receipt lacks its verification plan."); + for (const row of plan.rows) + await assertAccess( + "document", + row.documentId, + args.phase === "finalize" ? "editor" : "viewer", + ); + } + } + if (args.phase === "verify" || args.phase === "finalize") { + const replay = await db.transaction(async (tx) => { + const [receipt] = await tx + .select() + .from(schema.contentDatabaseMigrationReceipts) + .where( + and( + eq( + schema.contentDatabaseMigrationReceipts.databaseId, + args.databaseId, + ), + eq( + schema.contentDatabaseMigrationReceipts.idempotencyKey, + args.idempotencyKey, + ), + ), + ); + if (!receipt) throw new Error("Migration receipt not found."); + const storedResult = parseJson(receipt.resultJson); + const isTerminalReplay = + (args.phase === "verify" && receipt.state === "verified") || + (args.phase === "finalize" && receipt.state === "finalized"); + if (!isTerminalReplay) return null; + const expectedDigest = + args.phase === "verify" + ? receipt.postDigest + : storedResult.transitionExpectedPostDigest; + if (expectedDigest !== args.expectedPostDigest) + throw new Error( + "Expected post-migration digest does not match receipt.", + ); + if ( + snapshotDigest(await snapshotMigration(tx, args.databaseId)) !== + receipt.postDigest + ) + throw new Error( + args.phase === "verify" + ? "Migration has drifted; verification is refused." + : "Terminal migration result has drifted; replay is refused.", + ); + return receiptResult(receipt, true); + }); + if (replay) return replay; + } + let mutated = false; + const result = await db.transaction(async (tx) => { + if (args.phase !== "validate") { + await lockContentDatabaseMutation( + tx as unknown as ReturnType, + databaseId, + ); + await lockCurrentDatabaseMemberships(tx, databaseId); + } + if (args.phase === "validate" || args.phase === "apply") { + const planHash = digest(args.plan); + if (args.phase === "apply") { + const [existing] = await tx + .select() + .from(schema.contentDatabaseMigrationReceipts) + .where( + and( + eq( + schema.contentDatabaseMigrationReceipts.databaseId, + args.plan.databaseId, + ), + eq( + schema.contentDatabaseMigrationReceipts.idempotencyKey, + args.plan.idempotencyKey, + ), + ), + ); + if (existing) { + if (existing.planHash !== planHash) + throw new Error( + "Idempotency key was already used with a different migration plan.", + ); + if (existing.state !== "applied" && existing.state !== "verified") + throw new Error( + `Migration receipt is already ${existing.state}.`, + ); + const current = await snapshotMigration(tx, args.plan.databaseId); + if (snapshotDigest(current) !== existing.postDigest) + throw new Error( + "Applied migration has drifted; replay is refused.", + ); + return receiptResult(existing, true); + } + // A separate server can run the same migration while an editor is + // saving. Keep the durable database lock across the flush so that + // save is part of the state reloaded and validated by this writer. + if (flushUnderDurableLock) { + validatePlan( + args.plan, + await snapshotMigration(tx, args.plan.databaseId), + ); + await flushMigrationDocuments(args.plan.rows); + } + } + const snapshot = await snapshotMigration(tx, args.plan.databaseId); + validatePlan(args.plan, snapshot); + const preDigest = snapshotDigest(snapshot); + const orderedIds = snapshot.rows.map((row: any) => ({ + itemId: row.item.id, + documentId: row.document.id, + })); + if (args.phase === "validate") + return { + phase: "validate", + planHash, + preDigest, + counts: { + rows: snapshot.rows.length, + properties: args.plan.propertyDefinitions.length, + }, + orderedIds, + written: 0, + replayed: false, + verified: false, + }; + const receiptId = deterministicId( + "migration_receipt", + args.plan.databaseId, + args.plan.idempotencyKey, + ); + const now = new Date().toISOString(); + await tx.insert(schema.contentDatabaseMigrationReceipts).values({ + id: receiptId, + ownerEmail: snapshot.database.ownerEmail, + orgId: snapshot.database.orgId, + databaseId: args.plan.databaseId, + databaseDocumentId: args.plan.databaseDocumentId, + idempotencyKey: args.plan.idempotencyKey, + planHash, + state: "applying", + preDigest, + postDigest: preDigest, + rollbackJson: "{}", + resultJson: JSON.stringify({ phase: "apply", plan: args.plan }), + createdAt: now, + updatedAt: now, + }); + const rollback = await applyMigration( + tx, + args.plan, + snapshot, + receiptId, + now, + ); + const current = await snapshotMigration(tx, args.plan.databaseId); + const postDigest = snapshotDigest(current); + const resultJson = { + phase: "apply", + planHash, + legacyPropertyIds: args.plan.legacyPropertyIds, + counts: { + rows: args.plan.rows.length, + properties: args.plan.propertyDefinitions.length, + }, + orderedIds, + written: args.plan.rows.length, + verified: false, + plan: args.plan, + }; + const claimed = await tx + .update(schema.contentDatabaseMigrationReceipts) + .set({ + state: "applied", + postDigest, + rollbackJson: JSON.stringify(rollback), + resultJson: JSON.stringify(resultJson), + updatedAt: now, + }) + .where( + and( + eq(schema.contentDatabaseMigrationReceipts.id, receiptId), + eq(schema.contentDatabaseMigrationReceipts.state, "applying"), + ), + ) + .returning({ id: schema.contentDatabaseMigrationReceipts.id }); + if (claimed.length !== 1) + throw new Error("Migration receipt claim was lost."); + mutated = true; + return { + phase: "apply", + planHash, + legacyPropertyIds: args.plan.legacyPropertyIds, + counts: resultJson.counts, + orderedIds, + written: args.plan.rows.length, + verified: false, + receiptId, + state: "applied", + preDigest, + postDigest, + replayed: false, + }; + } + const [receipt] = await tx + .select() + .from(schema.contentDatabaseMigrationReceipts) + .where( + and( + eq( + schema.contentDatabaseMigrationReceipts.databaseId, + args.databaseId, + ), + eq( + schema.contentDatabaseMigrationReceipts.idempotencyKey, + args.idempotencyKey, + ), + ), + ); + if (!receipt) throw new Error("Migration receipt not found."); + const storedResult = parseJson(receipt.resultJson); + if (args.phase === "verify" && receipt.state === "verified") { + if (receipt.postDigest !== args.expectedPostDigest) + throw new Error( + "Expected post-migration digest does not match receipt.", + ); + if ( + snapshotDigest(await snapshotMigration(tx, args.databaseId)) !== + receipt.postDigest + ) + throw new Error("Migration has drifted; verification is refused."); + return receiptResult(receipt, true); + } + if ( + (args.phase === "rollback" && receipt.state === "rolled_back") || + (args.phase === "finalize" && receipt.state === "finalized") + ) { + if ( + storedResult.transitionExpectedPostDigest !== + args.expectedPostDigest + ) + throw new Error( + "Expected post-migration digest does not match receipt.", + ); + if ( + snapshotDigest(await snapshotMigration(tx, args.databaseId)) !== + receipt.postDigest + ) + throw new Error( + "Terminal migration result has drifted; replay is refused.", + ); + return receiptResult(receipt, true); + } + if (args.phase === "verify") { + if (receipt.state !== "applied" && receipt.state !== "verified") + throw new Error(`Migration receipt is already ${receipt.state}.`); + if (receipt.postDigest !== args.expectedPostDigest) + throw new Error( + "Expected post-migration digest does not match receipt.", + ); + const current = await snapshotMigration(tx, args.databaseId); + if (snapshotDigest(current) !== receipt.postDigest) + throw new Error("Migration has drifted; verification is refused."); + const plan = parseJson(receipt.resultJson).plan; + if (!plan) + throw new Error("Migration receipt lacks its verification plan."); + const newDefinitions = new Map( + plan.propertyDefinitions.map((definition: any) => [ + definition.id, + definition, + ]), + ); + if ( + current.definitions.filter((definition: any) => + newDefinitions.has(definition.id), + ).length !== plan.propertyDefinitions.length + ) + throw new Error( + "Migration verification found a property definition count mismatch.", + ); + for (const definition of plan.propertyDefinitions) { + const actual = current.definitions.find( + (candidate: any) => candidate.id === definition.id, + ); + const optionsJson = JSON.stringify( + definition.type === "multi_select" + ? { options: definition.options } + : {}, + ); + if ( + !actual || + actual.name !== definition.name.trim() || + actual.type !== definition.type || + actual.visibility !== definition.visibility || + actual.optionsJson !== optionsJson + ) + throw new Error( + "Migration verification found a property definition mismatch.", + ); + } + for (const row of plan.rows) { + const persisted = current.rows.find( + (candidate: any) => + candidate.item.id === row.itemId && + candidate.document.id === row.documentId, + ); + if (!persisted || persisted.document.content !== row.content) + throw new Error( + "Migration verification found a row body mismatch.", + ); + for (const value of row.propertyValues) { + const actual = current.values.find( + (candidate: any) => + candidate.documentId === row.documentId && + candidate.propertyId === value.propertyId, + )?.valueJson; + if ( + actual !== + serializeMigrationValue( + newDefinitions.get(value.propertyId) as any, + value.value, + ) + ) + throw new Error( + "Migration verification found a new property mismatch.", + ); + } + for (const protectedValue of row.protectedPropertyValues) { + const actual = + current.values.find( + (candidate: any) => + candidate.documentId === row.documentId && + candidate.propertyId === protectedValue.propertyId, + )?.valueJson ?? "null"; + if (actual !== protectedValue.valueJson) + throw new Error( + "Migration verification found a protected property mismatch.", + ); + } + } + const now = new Date().toISOString(); + const resultJson = { + ...parseJson(receipt.resultJson), + phase: "verify", + verified: true, + }; + const transitioned = await tx + .update(schema.contentDatabaseMigrationReceipts) + .set({ + state: "verified", + resultJson: JSON.stringify(resultJson), + updatedAt: now, + }) + .where( + and( + eq(schema.contentDatabaseMigrationReceipts.id, receipt.id), + eq(schema.contentDatabaseMigrationReceipts.state, "applied"), + ), + ) + .returning({ id: schema.contentDatabaseMigrationReceipts.id }); + if (transitioned.length !== 1) + throw new Error("Migration receipt state changed during verify."); + mutated = true; + return { + receiptId: receipt.id, + state: "verified", + preDigest: receipt.preDigest, + postDigest: receipt.postDigest, + replayed: false, + verified: true, + }; + } + if (args.phase === "finalize" && receipt.state !== "verified") + throw new Error( + "Migration receipt must be verified before finalization.", + ); + if ( + args.phase === "rollback" && + receipt.state !== "applied" && + receipt.state !== "verified" + ) + throw new Error(`Migration receipt is already ${receipt.state}.`); + if (receipt.postDigest !== args.expectedPostDigest) + throw new Error( + "Expected post-migration digest does not match receipt.", + ); + if (args.phase === "rollback" && flushUnderDurableLock) { + await flushMigrationDocuments( + parseJson(receipt.rollbackJson).versions ?? [], + ); + } + const current = await snapshotMigration(tx, args.databaseId); + if (snapshotDigest(current) !== receipt.postDigest) + throw new Error( + "Migration has drifted; guarded operation is refused.", + ); + const rollback = parseJson(receipt.rollbackJson); + const now = new Date().toISOString(); + if (args.phase === "rollback") { + for (const prior of rollback.versions ?? []) { + const [version] = await tx + .select() + .from(schema.documentVersions) + .where(eq(schema.documentVersions.id, prior.versionId)); + if (!version) throw new Error("Rollback snapshot is missing."); + const restoredRows = await tx + .update(schema.documents) + .set({ + title: version.title, + content: version.content, + updatedAt: now, + }) + .where( + and( + eq(schema.documents.id, prior.documentId), + eq(schema.documents.updatedAt, prior.appliedUpdatedAt), + ), + ) + .returning({ id: schema.documents.id }); + if (restoredRows.length !== 1) + throw new Error( + `Rollback row ${prior.documentId} changed concurrently.`, + ); + } + const ids = rollback.createdPropertyIds ?? []; + if (ids.length) { + await tx + .delete(schema.documentPropertyValues) + .where(inArray(schema.documentPropertyValues.propertyId, ids)); + await tx + .delete(schema.documentPropertyDefinitions) + .where(inArray(schema.documentPropertyDefinitions.id, ids)); + } + await tx + .update(schema.contentDatabases) + .set({ updatedAt: now }) + .where(eq(schema.contentDatabases.id, args.databaseId)); + const restored = await snapshotMigration(tx, args.databaseId); + const postDigest = snapshotDigest(restored); + if (postDigest !== receipt.preDigest) + throw new Error("Rollback verification failed."); + const resultJson = { + phase: "rollback", + transitionExpectedPostDigest: receipt.postDigest, + counts: { + rows: (rollback.versions ?? []).length, + properties: ids.length, + }, + verified: true, + }; + const transitioned = await tx + .update(schema.contentDatabaseMigrationReceipts) + .set({ + state: "rolled_back", + postDigest, + resultJson: JSON.stringify(resultJson), + updatedAt: now, + }) + .where( + and( + eq(schema.contentDatabaseMigrationReceipts.id, receipt.id), + inArray(schema.contentDatabaseMigrationReceipts.state, [ + "applied", + "verified", + ]), + ), + ) + .returning({ id: schema.contentDatabaseMigrationReceipts.id }); + if (transitioned.length !== 1) + throw new Error("Migration receipt state changed during rollback."); + mutated = true; + return { + receiptId: receipt.id, + state: "rolled_back", + preDigest: receipt.preDigest, + postDigest, + counts: { + rows: (rollback.versions ?? []).length, + properties: ids.length, + }, + replayed: false, + verified: true, + }; + } + const planResult = parseJson(receipt.resultJson); + // Legacy ids are deliberately copied into the receipt result only after apply validation. + const legacyIds: string[] = planResult.legacyPropertyIds ?? []; + const legacy = current.definitions.filter((definition: any) => + legacyIds.includes(definition.id), + ); + if ( + legacy.length !== legacyIds.length || + legacy.some( + (definition: any) => + definition.systemRole || definition.type === "blocks", + ) + ) + throw new Error("Legacy property is missing or unsafe to finalize."); + if (legacyIds.length) { + await tx + .delete(schema.documentPropertyValues) + .where( + inArray(schema.documentPropertyValues.propertyId, legacyIds), + ); + await tx + .delete(schema.documentPropertyDefinitions) + .where(inArray(schema.documentPropertyDefinitions.id, legacyIds)); + } + await touchContentDatabase( + tx as unknown as ReturnType, + args.databaseId, + now, + ); + const finalized = await snapshotMigration(tx, args.databaseId); + if ( + finalized.definitions.some((definition: any) => + legacyIds.includes(definition.id), + ) || + finalized.values.some((value: any) => + legacyIds.includes(value.propertyId), + ) + ) + throw new Error("Legacy property finalization verification failed."); + const postDigest = snapshotDigest(finalized); + const resultJson = { + phase: "finalize", + transitionExpectedPostDigest: receipt.postDigest, + counts: { rows: 0, properties: legacyIds.length }, + verified: true, + }; + const transitioned = await tx + .update(schema.contentDatabaseMigrationReceipts) + .set({ + state: "finalized", + postDigest, + resultJson: JSON.stringify(resultJson), + updatedAt: now, + }) + .where( + and( + eq(schema.contentDatabaseMigrationReceipts.id, receipt.id), + eq(schema.contentDatabaseMigrationReceipts.state, "verified"), + ), + ) + .returning({ id: schema.contentDatabaseMigrationReceipts.id }); + if (transitioned.length !== 1) + throw new Error("Migration receipt state changed during finalize."); + mutated = true; + return { + receiptId: receipt.id, + state: "finalized", + preDigest: receipt.preDigest, + postDigest, + counts: { rows: 0, properties: legacyIds.length }, + replayed: false, + verified: true, + }; + }); + if (mutated) + await writeAppState("refresh-signal", { ts: Date.now() }).catch(() => { + // The receipt is already committed; polling reconciles this optional + // UI hint when a concurrent SQLite writer briefly holds the database. + }); + return result; + }); + }, +}); diff --git a/templates/content/actions/move-database-item.ts b/templates/content/actions/move-database-item.ts index 751c8de55a..7c750cb40f 100644 --- a/templates/content/actions/move-database-item.ts +++ b/templates/content/actions/move-database-item.ts @@ -5,6 +5,10 @@ import { and, asc, eq, inArray, isNull, sql } from "drizzle-orm"; import { z } from "zod"; import { getDb, schema } from "../server/db/index.js"; +import { + lockContentDatabaseMutation, + touchContentDatabase, +} from "./_content-database-mutation-lock.js"; import { getContentDatabaseResponse } from "./_database-utils.js"; import { databaseItemsPositionScope, @@ -69,6 +73,11 @@ export default defineAction({ databaseItemsPositionScope(row.item.databaseId), () => db.transaction(async (tx) => { + await lockContentDatabaseMutation( + tx as unknown as ReturnType, + row.item.databaseId, + ); + const items = await tx .select() .from(schema.contentDatabaseItems) @@ -88,6 +97,11 @@ export default defineAction({ const [moved] = nextItems.splice(currentIndex, 1); nextItems.splice(nextIndex, 0, moved); const now = new Date().toISOString(); + await touchContentDatabase( + tx as unknown as ReturnType, + row.item.databaseId, + now, + ); const itemIds = nextItems.map((item) => item.id); const documentIds = nextItems.map((item) => item.documentId); diff --git a/templates/content/actions/permanently-delete-document.ts b/templates/content/actions/permanently-delete-document.ts index 6911944922..889b10e15d 100644 --- a/templates/content/actions/permanently-delete-document.ts +++ b/templates/content/actions/permanently-delete-document.ts @@ -4,7 +4,12 @@ import { assertAccess } from "@agent-native/core/sharing"; import { z } from "zod"; import { getDb } from "../server/db/index.js"; -import { deleteTrashedDocumentSubtree } from "./delete-document.js"; +import { + deleteTrashedDocumentSubtree, + PermanentDeleteScopeChangedError, +} from "./delete-document.js"; + +const MAX_DELETE_SCOPE_ATTEMPTS = 3; export default defineAction({ description: @@ -15,11 +20,27 @@ export default defineAction({ run: async ({ id }) => { const access = await assertAccess("document", id, "admin"); const db = getDb(); - const deleted = await deleteTrashedDocumentSubtree( - db, - id, - access.resource.ownerEmail as string, - ); + let deleted: string[] | undefined; + for (let attempt = 1; attempt <= MAX_DELETE_SCOPE_ATTEMPTS; attempt += 1) { + try { + deleted = await db.transaction((tx) => + deleteTrashedDocumentSubtree( + tx as unknown as ReturnType, + id, + access.resource.ownerEmail as string, + ), + ); + break; + } catch (error) { + if ( + !(error instanceof PermanentDeleteScopeChangedError) || + attempt === MAX_DELETE_SCOPE_ATTEMPTS + ) { + throw error; + } + } + } + if (!deleted) throw new Error("Document deletion did not complete."); await writeAppState("refresh-signal", { ts: Date.now() }); return { success: true, deleted: deleted.length }; }, diff --git a/templates/content/actions/remove-database-items.ts b/templates/content/actions/remove-database-items.ts index cfa0077da3..14fb76e312 100644 --- a/templates/content/actions/remove-database-items.ts +++ b/templates/content/actions/remove-database-items.ts @@ -4,6 +4,10 @@ import { assertAccess } from "@agent-native/core/sharing"; import { and, eq, inArray } from "drizzle-orm"; import { getDb, schema } from "../server/db/index.js"; +import { + lockContentDatabaseMutation, + touchContentDatabase, +} from "./_content-database-mutation-lock.js"; import { assertNotWorkspaceCatalogDocuments } from "./_content-space-catalog-guards.js"; import { lockDatabaseMemberships } from "./_database-membership-lock.js"; import { @@ -80,6 +84,15 @@ export default defineAction({ await withPositionLock(databaseItemsPositionScope(database.id), () => db.transaction(async (tx) => { + await lockContentDatabaseMutation( + tx as unknown as ReturnType, + database.id, + ); + await touchContentDatabase( + tx as unknown as ReturnType, + database.id, + now, + ); if (removedItemIds.length > 0) { await lockDatabaseMemberships(tx, removedItemIds); const [sourceRow, hydrationRow] = await Promise.all([ diff --git a/templates/content/actions/reorder-document-property.ts b/templates/content/actions/reorder-document-property.ts index a515c723c0..c89a41c84f 100644 --- a/templates/content/actions/reorder-document-property.ts +++ b/templates/content/actions/reorder-document-property.ts @@ -5,6 +5,7 @@ import { and, asc, eq } from "drizzle-orm"; import { z } from "zod"; import { getDb, schema } from "../server/db/index.js"; +import { lockContentDatabaseMutation } from "./_content-database-mutation-lock.js"; import { listPropertiesForDocument, resolvePropertyDatabaseForDocument, @@ -45,54 +46,52 @@ export default defineAction({ ); if (!database) throw new Error("Document is not part of a database."); - const definitions = await db - .select() - .from(schema.documentPropertyDefinitions) - .where(eq(schema.documentPropertyDefinitions.databaseId, database.id)) - .orderBy(asc(schema.documentPropertyDefinitions.position)); - - const ids = definitions.map((definition) => definition.id); - if (!ids.includes(propertyId)) { - throw new Error(`Property "${propertyId}" not found`); - } - if (!ids.includes(targetPropertyId)) { - throw new Error(`Property "${targetPropertyId}" not found`); - } - const definitionById = new Map( - definitions.map((definition) => [definition.id, definition]), - ); - if ( - definitionById.get(propertyId)?.systemRole || - definitionById.get(targetPropertyId)?.systemRole - ) { - throw new Error("System properties cannot be reordered."); - } - if (propertyId === targetPropertyId) { - return { - documentId, - databaseId: database.id, - properties: await listPropertiesForDocument(document, database.id), - }; - } + const now = new Date().toISOString(); + await db.transaction(async (tx) => { + await lockContentDatabaseMutation( + tx as unknown as ReturnType, + database.id, + ); + const definitions = await tx + .select() + .from(schema.documentPropertyDefinitions) + .where(eq(schema.documentPropertyDefinitions.databaseId, database.id)) + .orderBy(asc(schema.documentPropertyDefinitions.position)); + const ids = definitions.map((definition) => definition.id); + if (!ids.includes(propertyId)) { + throw new Error(`Property "${propertyId}" not found`); + } + if (!ids.includes(targetPropertyId)) { + throw new Error(`Property "${targetPropertyId}" not found`); + } + const definitionById = new Map( + definitions.map((definition) => [definition.id, definition]), + ); + if ( + definitionById.get(propertyId)?.systemRole || + definitionById.get(targetPropertyId)?.systemRole + ) { + throw new Error("System properties cannot be reordered."); + } + if (propertyId === targetPropertyId) return; - // Rebuild the order with the moved property re-inserted relative to target. - const remaining = ids.filter((id) => id !== propertyId); - const targetIndex = remaining.indexOf(targetPropertyId); - const insertAt = position === "after" ? targetIndex + 1 : targetIndex; - remaining.splice(insertAt, 0, propertyId); + const remaining = ids.filter((id) => id !== propertyId); + const targetIndex = remaining.indexOf(targetPropertyId); + const insertAt = position === "after" ? targetIndex + 1 : targetIndex; + remaining.splice(insertAt, 0, propertyId); - const now = new Date().toISOString(); - for (let index = 0; index < remaining.length; index += 1) { - await db - .update(schema.documentPropertyDefinitions) - .set({ position: index, updatedAt: now }) - .where( - and( - eq(schema.documentPropertyDefinitions.id, remaining[index]), - eq(schema.documentPropertyDefinitions.databaseId, database.id), - ), - ); - } + for (let index = 0; index < remaining.length; index += 1) { + await tx + .update(schema.documentPropertyDefinitions) + .set({ position: index, updatedAt: now }) + .where( + and( + eq(schema.documentPropertyDefinitions.id, remaining[index]), + eq(schema.documentPropertyDefinitions.databaseId, database.id), + ), + ); + } + }); await writeAppState("refresh-signal", { ts: Date.now() }); diff --git a/templates/content/actions/set-document-property.ts b/templates/content/actions/set-document-property.ts index d6d00098bf..cafd59097f 100644 --- a/templates/content/actions/set-document-property.ts +++ b/templates/content/actions/set-document-property.ts @@ -1,6 +1,6 @@ import { defineAction } from "@agent-native/core"; import { assertAccess } from "@agent-native/core/sharing"; -import { and, eq, isNull, ne, sql } from "drizzle-orm"; +import { and, eq, isNull, ne } from "drizzle-orm"; import { z } from "zod"; import { getDb, schema } from "../server/db/index.js"; @@ -12,6 +12,7 @@ import { parsePropertyOptions, type DocumentPropertyType, } from "../shared/properties.js"; +import { lockContentDatabaseMutation } from "./_content-database-mutation-lock.js"; import { resolveContentDocumentAccess } from "./_content-document-access.js"; import { lockDatabaseMemberships } from "./_database-membership-lock.js"; import { @@ -80,11 +81,41 @@ export default defineAction({ await assertAccess("document", documentId, "editor"); const normalized = normalizePropertyValue(type, value); const content = typeof normalized === "string" ? normalized : ""; - const target = blocksStorageTarget( + let target = blocksStorageTarget( parsePropertyOptions(definition.optionsJson), ); await db.transaction(async (tx) => { await lockDatabaseMemberships(tx, [membership.id]); + const [lockedDefinition] = await tx + .select() + .from(schema.documentPropertyDefinitions) + .where(eq(schema.documentPropertyDefinitions.id, propertyId)); + const [lockedMembership] = await tx + .select({ id: schema.contentDatabaseItems.id }) + .from(schema.contentDatabaseItems) + .where( + and( + eq(schema.contentDatabaseItems.id, membership.id), + eq(schema.contentDatabaseItems.databaseId, database.id), + eq(schema.contentDatabaseItems.documentId, documentId), + ), + ); + if (!lockedDefinition || lockedDefinition.databaseId !== database.id) { + throw new Error(`Property "${propertyId}" not found`); + } + if (!lockedMembership) { + throw new Error("Document is not part of this database."); + } + if ( + !isBlocksPropertyType(lockedDefinition.type as DocumentPropertyType) + ) { + throw new Error( + "Property type changed before the operation completed.", + ); + } + target = blocksStorageTarget( + parsePropertyOptions(lockedDefinition.optionsJson), + ); if (target === "document_body") { await tx .update(schema.documents) @@ -130,9 +161,13 @@ export default defineAction({ const valueJson = normalizedValueJson(type, value); await db.transaction(async (tx) => { + await lockContentDatabaseMutation( + tx as unknown as ReturnType, + database.id, + ); const [lockedDatabase] = await tx - .update(schema.contentDatabases) - .set({ updatedAt: sql`${schema.contentDatabases.updatedAt}` }) + .select({ id: schema.contentDatabases.id }) + .from(schema.contentDatabases) .where( and( eq(schema.contentDatabases.id, database.id), @@ -140,38 +175,43 @@ export default defineAction({ eq(schema.contentDatabases.ownerEmail, database.ownerEmail), isNull(schema.contentDatabases.deletedAt), ), - ) - .returning({ id: schema.contentDatabases.id }); + ); if (!lockedDatabase) throw new Error("Database is no longer active."); + await lockDatabaseMemberships(tx, [membership.id]); const [lockedDefinition] = await tx - .update(schema.documentPropertyDefinitions) - .set({ - updatedAt: sql`${schema.documentPropertyDefinitions.updatedAt}`, - }) + .select() + .from(schema.documentPropertyDefinitions) + .where(eq(schema.documentPropertyDefinitions.id, propertyId)); + const [lockedMembership] = await tx + .select({ id: schema.contentDatabaseItems.id }) + .from(schema.contentDatabaseItems) .where( and( - eq(schema.documentPropertyDefinitions.id, propertyId), - eq(schema.documentPropertyDefinitions.databaseId, database.id), - eq( - schema.documentPropertyDefinitions.ownerEmail, - database.ownerEmail, - ), + eq(schema.contentDatabaseItems.id, membership.id), + eq(schema.contentDatabaseItems.databaseId, database.id), + eq(schema.contentDatabaseItems.documentId, documentId), ), - ) - .returning({ - type: schema.documentPropertyDefinitions.type, - systemRole: schema.documentPropertyDefinitions.systemRole, - }); - if ( - !lockedDefinition || - lockedDefinition.type !== definition.type || - lockedDefinition.systemRole - ) { + ); + if (!lockedDefinition || lockedDefinition.databaseId !== database.id) { + throw new Error(`Property "${propertyId}" not found`); + } + if (!lockedMembership) { + throw new Error("Document is not part of this database."); + } + const lockedType = lockedDefinition.type as DocumentPropertyType; + if (lockedType !== type) { throw new Error( - `Property "${propertyId}" changed or was deleted before its value could be written.`, + "Property type changed before the operation completed.", ); } - await lockDatabaseMemberships(tx, [membership.id]); + if (isBlocksPropertyType(lockedType)) { + throw new Error( + "Property type changed before the operation completed.", + ); + } + if (isComputedPropertyType(lockedType)) { + throw new Error("Computed properties cannot be edited."); + } const [conflictingClaim] = await tx .select({ id: schema.contentDatabaseItemKeyClaims.id }) .from(schema.contentDatabaseItemKeyClaims) diff --git a/templates/content/actions/submit-content-database-form.ts b/templates/content/actions/submit-content-database-form.ts index 8fb0d97142..aad7e77e17 100644 --- a/templates/content/actions/submit-content-database-form.ts +++ b/templates/content/actions/submit-content-database-form.ts @@ -25,6 +25,10 @@ import { type DocumentPropertyType, type DocumentPropertyValue, } from "../shared/properties.js"; +import { + lockContentDatabaseMutation, + touchContentDatabase, +} from "./_content-database-mutation-lock.js"; import { ensureDocumentFilesMembership } from "./_content-files.js"; import { nanoid, parseDatabaseViewConfig } from "./_property-utils.js"; @@ -47,6 +51,21 @@ const submitContentDatabaseFormSchema = z.object({ type PropertyDefinitionRow = typeof schema.documentPropertyDefinitions.$inferSelect; +function propertyDefinitionFingerprint(definitions: PropertyDefinitionRow[]) { + return JSON.stringify( + definitions + .map((definition) => ({ + id: definition.id, + name: definition.name, + type: definition.type, + systemRole: definition.systemRole, + visibility: definition.visibility, + optionsJson: definition.optionsJson, + })) + .sort((left, right) => left.id.localeCompare(right.id)), + ); +} + function resolveFormView( views: ContentDatabaseView[], activeViewId: string, @@ -231,6 +250,7 @@ export default defineAction({ ), ), ); + const definitionsFingerprint = propertyDefinitionFingerprint(definitions); const viewConfig = parseDatabaseViewConfig(database.viewConfigJson); const formView = resolveFormView( viewConfig.views, @@ -310,6 +330,35 @@ export default defineAction({ const createdBy = getRequestUserEmail() ?? database.ownerEmail; await db.transaction(async (tx) => { + await lockContentDatabaseMutation( + tx as unknown as ReturnType, + databaseId, + ); + await touchContentDatabase( + tx as unknown as ReturnType, + databaseId, + now, + ); + const lockedDefinitions = await tx + .select() + .from(schema.documentPropertyDefinitions) + .where( + and( + eq(schema.documentPropertyDefinitions.databaseId, databaseId), + eq( + schema.documentPropertyDefinitions.ownerEmail, + database.ownerEmail, + ), + ), + ); + if ( + propertyDefinitionFingerprint(lockedDefinitions) !== + definitionsFingerprint + ) { + throw new Error( + "Database properties changed before form submission completed.", + ); + } const [maxDocumentPosition] = await tx .select({ max: sql`COALESCE(MAX(position), -1)` }) .from(schema.documents) diff --git a/templates/content/changelog/2026-08-01-content-can-reorganize-every-row-in-an-existing-database-as-.md b/templates/content/changelog/2026-08-01-content-can-reorganize-every-row-in-an-existing-database-as-.md new file mode 100644 index 0000000000..ed5e803997 --- /dev/null +++ b/templates/content/changelog/2026-08-01-content-can-reorganize-every-row-in-an-existing-database-as-.md @@ -0,0 +1,6 @@ +--- +type: added +date: 2026-08-01 +--- + +Content can reorganize every row in an existing database as one verified, reversible operation. diff --git a/templates/content/parity/__tests__/database-row-batch-reliability.test.ts b/templates/content/parity/__tests__/database-row-batch-reliability.test.ts index 9676978b91..50cd388e32 100644 --- a/templates/content/parity/__tests__/database-row-batch-reliability.test.ts +++ b/templates/content/parity/__tests__/database-row-batch-reliability.test.ts @@ -60,6 +60,9 @@ describe("database row batch reliability", () => { "Use this for two or more selected/named rows instead of looping duplicate-database-item", ); expect(removeBatchActionSource).toContain("without deleting the pages"); + expect(removeBatchActionSource).toContain( + "await lockContentDatabaseMutation(", + ); expect(singularDuplicateActionSource).toContain( "For two or more rows, use duplicate-database-items once instead of looping this action", ); diff --git a/templates/content/parity/matrix.md b/templates/content/parity/matrix.md index 020d77e999..30543ed59f 100644 --- a/templates/content/parity/matrix.md +++ b/templates/content/parity/matrix.md @@ -9,7 +9,7 @@ This generated matrix tracks whether high-value Content UI operations use the sa | database.lifecycle-and-trash | database | Create, soft-delete, restore, list, and inspect content databases | action-backed | `create-content-database`, `create-inline-content-database`, `delete-content-database`, `get-content-database`, `list-content-databases`, `list-trashed-content-databases`, `restore-content-database` | `app/components/editor/SlashCommandMenu.tsx`, `app/hooks/use-content-database.ts`, `app/hooks/use-documents.ts` | Database pages and database records are created, read, soft-deleted, restored, and listed. | - | - | P0 | covered | `actions/content-database-lifecycle.db.test.ts` | `database-source-scope` | - | | database.private-preview-drafts | database | Persist and reconcile a user's private database-page preview draft | action-backed | `get-preview-document-draft`, `update-preview-document-draft` | `app/components/editor/database/DatabaseView.tsx`, `app/hooks/use-documents.ts` | A user's private preview draft is read, saved, conflict-checked, or deleted without changing the shared database page until the normal save flow applies it. | These per-user editor-state actions are intentionally hidden from agent tools because preview drafts are a private UI recovery mechanism. | - | P1 | covered | `actions/preview-document-draft.db.test.ts` | - | - | | database.properties-and-view-config | database | Configure properties, values, ordering, and saved views | action-backed | `configure-document-property`, `delete-document-property`, `duplicate-document-property`, `get-content-database-personal-view`, `list-document-properties`, `reorder-document-property`, `set-document-property`, `update-content-database-personal-view`, `update-content-database-view` | `app/components/editor/DocumentProperties.tsx`, `app/components/editor/DocumentDatabase.tsx`, `app/components/editor/database/DatabaseView.tsx`, `app/hooks/use-document-properties.ts` | Property schemas, property values, property order, filters, sorts, grouping, hidden columns, view type, and view settings are stored. | - | - | P0 | covered | `actions/bind-content-database-source-field.db.test.ts`, `actions/content-database-source-actions.test.ts`, `actions/resync-content-database-source.db.test.ts` | `database-source-scope` | - | -| database.rows | database | Add, duplicate, move, open, edit, and remove database rows | action-backed | `add-database-item`, `upsert-database-item-by-key`, `remove-database-items`, `duplicate-database-items`, `duplicate-database-item`, `move-database-item`, `set-document-property` | `app/components/editor/DocumentDatabase.tsx`, `app/components/editor/database/DatabaseView.tsx` | Database row memberships and ordering are created, duplicated, moved, edited, and removed without deleting the backing page. | - | - | P0 | covered | `actions/database-row-batch-actions.db.test.ts`, `parity/__tests__/database-row-batch-reliability.test.ts` | `database-bulk-row-reliability` | - | +| database.rows | database | Add, duplicate, move, open, edit, and remove database rows | action-backed | `add-database-item`, `upsert-database-item-by-key`, `remove-database-items`, `duplicate-database-items`, `duplicate-database-item`, `migrate-content-database-rows`, `move-database-item`, `set-document-property` | `app/components/editor/DocumentDatabase.tsx`, `app/components/editor/database/DatabaseView.tsx` | Database row memberships and ordering are created, duplicated, moved, edited, and removed without deleting the backing page; bounded migrations atomically update row bodies and properties through the same canonical data model. | - | - | P0 | covered | `actions/database-row-batch-actions.db.test.ts`, `actions/migrate-content-database-rows.db.test.ts`, `parity/__tests__/database-row-batch-reliability.test.ts` | `database-bulk-row-reliability` | - | | database.table-query-page | database | Query one constrained page while retaining database metadata | action-backed | `query-content-database-items` | `app/components/editor/database/DatabaseView.tsx`, `app/hooks/use-content-database.ts` | - | This UI-only bounded projection is intentionally hidden with agentTool: false; agents use get-content-database for the complete database contract. | - | P0 | covered | `actions/content-database-lifecycle.db.test.ts`, `app/hooks/use-content-database.test.ts` | - | - | | editor.agent-assist-prompts | editor | Ask AI from slash generation or comment context | client-assist | - | `app/components/editor/SlashCommandMenu.tsx`, `app/components/editor/CommentsSidebar.tsx` | No direct durable mutation; the prompt asks the agent to use document actions when it decides to write. | - | - | P1 | none | - | - | - | | editor.client-formatting-and-insertions | editor | Rich text formatting, selection state, slash block insertion, and copy actions | client-only-ephemeral | - | `app/components/editor/BubbleToolbar.tsx`, `app/components/editor/SlashCommandMenu.tsx`, `app/components/editor/DocumentToolbar.tsx` | - | - | - | P1 | none | - | - | - | diff --git a/templates/content/parity/matrix.ts b/templates/content/parity/matrix.ts index d75fb63dfb..307d8bdb93 100644 --- a/templates/content/parity/matrix.ts +++ b/templates/content/parity/matrix.ts @@ -317,9 +317,9 @@ export const parityMatrix: ParityRow[] = [ "app/components/editor/database/DatabaseView.tsx", ], durableEffect: - "Database row memberships and ordering are created, duplicated, moved, edited, and removed without deleting the backing page.", + "Database row memberships and ordering are created, duplicated, moved, edited, and removed without deleting the backing page; bounded migrations atomically update row bodies and properties through the same canonical data model.", uiImplementation: - "Row controls call row actions; selected-row duplicate/removal call bounded batch actions, while bulk property edits remain a later reliability slice.", + "Row controls call row actions; selected-row duplicate/removal call bounded batch actions, while bounded whole-database schema-and-body migrations use one validated, receipt-backed action instead of many partial writes.", status: "action-backed", actions: [ "add-database-item", @@ -327,6 +327,7 @@ export const parityMatrix: ParityRow[] = [ "remove-database-items", "duplicate-database-items", "duplicate-database-item", + "migrate-content-database-rows", "move-database-item", "set-document-property", ], @@ -337,6 +338,7 @@ export const parityMatrix: ParityRow[] = [ followUpPR: null, coverageRefs: [ "actions/database-row-batch-actions.db.test.ts", + "actions/migrate-content-database-rows.db.test.ts", "parity/__tests__/database-row-batch-reliability.test.ts", ], evalScenarioIds: ["database-bulk-row-reliability"], diff --git a/templates/content/server/__tests__/db.spec.ts b/templates/content/server/__tests__/db.spec.ts index 218c6450f8..d1d3b4011f 100644 --- a/templates/content/server/__tests__/db.spec.ts +++ b/templates/content/server/__tests__/db.spec.ts @@ -108,6 +108,21 @@ describe("content database migrations", () => { ); }); + it("creates bounded database migration receipts additively", () => { + const source = readFileSync( + join(__dirname, "..", "plugins", "db.ts"), + "utf8", + ); + + expect(source).toContain( + "CREATE TABLE IF NOT EXISTS content_database_migration_receipts", + ); + expect(source).toContain( + "content_database_migration_receipts_database_key_unique", + ); + expect(source).toContain('name: "content-database-migration-receipts"'); + }); + it("creates Builder MDX sidecar cache table additively", () => { const source = readFileSync( join(__dirname, "..", "plugins", "db.ts"), @@ -166,5 +181,6 @@ describe("content database migrations", () => { expect(changeSetDelete).toBeGreaterThan(-1); expect(executionDelete).toBeLessThan(changeSetDelete); expect(reviewDelete).toBeLessThan(changeSetDelete); + expect(source).toContain("delete(schema.contentDatabaseMigrationReceipts)"); }); }); diff --git a/templates/content/server/db/schema.ts b/templates/content/server/db/schema.ts index aa2fe94a6d..ba4ab76113 100644 --- a/templates/content/server/db/schema.ts +++ b/templates/content/server/db/schema.ts @@ -464,6 +464,36 @@ export const contentDatabaseSourceExecutionClaims = table( }, ); +export const contentDatabaseMigrationReceipts = table( + "content_database_migration_receipts", + { + id: text("id").primaryKey(), + ownerEmail: text("owner_email").notNull().default("local@localhost"), + orgId: text("org_id"), + databaseId: text("database_id").notNull(), + databaseDocumentId: text("database_document_id").notNull(), + idempotencyKey: text("idempotency_key").notNull(), + planHash: text("plan_hash").notNull(), + state: text("state").notNull(), + preDigest: text("pre_digest").notNull(), + postDigest: text("post_digest").notNull(), + rollbackJson: text("rollback_json").notNull().default("{}"), + resultJson: text("result_json").notNull().default("{}"), + createdAt: text("created_at").notNull().default(now()), + updatedAt: text("updated_at").notNull().default(now()), + }, + (receipt) => [ + uniqueIndex("content_database_migration_receipts_database_key_unique").on( + receipt.databaseId, + receipt.idempotencyKey, + ), + index("content_database_migration_receipts_owner_database_idx").on( + receipt.ownerEmail, + receipt.databaseId, + ), + ], +); + export const documentPropertyValues = table("document_property_values", { id: text("id").primaryKey(), ownerEmail: text("owner_email").notNull().default("local@localhost"), diff --git a/templates/content/server/plugins/db.ts b/templates/content/server/plugins/db.ts index 558e88c5bb..0b44ff46d9 100644 --- a/templates/content/server/plugins/db.ts +++ b/templates/content/server/plugins/db.ts @@ -932,6 +932,30 @@ const runContentMigrations = runMigrations( name: "content-database-item-stable-key-single-active-claim", sql: `CREATE UNIQUE INDEX IF NOT EXISTS content_database_item_key_claims_database_property_document_unique ON content_database_item_key_claims (database_id, property_id, document_id)`, }, + { + version: 80, + name: "content-database-migration-receipts", + sql: `CREATE TABLE IF NOT EXISTS content_database_migration_receipts ( + id TEXT PRIMARY KEY, + owner_email TEXT NOT NULL DEFAULT 'local@localhost', + org_id TEXT, + database_id TEXT NOT NULL, + database_document_id TEXT NOT NULL, + idempotency_key TEXT NOT NULL, + plan_hash TEXT NOT NULL, + state TEXT NOT NULL, + pre_digest TEXT NOT NULL, + post_digest TEXT NOT NULL, + rollback_json TEXT NOT NULL DEFAULT '{}', + result_json TEXT NOT NULL DEFAULT '{}', + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) + ); + CREATE UNIQUE INDEX IF NOT EXISTS content_database_migration_receipts_database_key_unique + ON content_database_migration_receipts (database_id, idempotency_key); + CREATE INDEX IF NOT EXISTS content_database_migration_receipts_owner_database_idx + ON content_database_migration_receipts (owner_email, database_id)`, + }, ], { table: "content_migrations" }, );