fix(workspaces): retry transient Durable Object storage resets

src/features/workspaces/kernel/workspace-kernel-storage.test.ts

@@ -0,0 +1,77 @@
+import { describe, expect, it, vi } from "vitest";
+
+import {
+	isTransientStorageError,
+	runKernelSqlWithRetry,
+} from "#/features/workspaces/kernel/workspace-kernel-storage";
+
+class SqlError extends Error {
+	constructor(cause: unknown) {
+		const message = cause instanceof Error ? cause.message : String(cause);
+		super(`SQL query failed: ${message}`);
+		this.name = "SqlError";
+		this.cause = cause;
+	}
+}
+
+describe("workspace kernel storage transient errors", () => {
+	it("classifies a mid-query Durable Object reset as transient", () => {
+		const error = new SqlError(
+			new Error("Internal error in Durable Object storage caused object to be reset"),
+		);
+
+		expect(isTransientStorageError(error)).toBe(true);
+	});
+
+	it("classifies the underlying reset error directly", () => {
+		const error = new Error("Internal error in Durable Object storage caused object to be reset");
+
+		expect(isTransientStorageError(error)).toBe(true);
+	});
+
+	it("does not classify a genuine query defect as transient", () => {
+		const error = new SqlError(new Error("no such column: bogus"));
+
+		expect(isTransientStorageError(error)).toBe(false);
+	});
+
+	it("tolerates non-error values", () => {
+		expect(isTransientStorageError("boom")).toBe(false);
+		expect(isTransientStorageError(undefined)).toBe(false);
+	});
+
+	it("retries a transient reset and returns once it succeeds", () => {
+		const run = vi
+			.fn<() => string>()
+			.mockImplementationOnce(() => {
+				throw new SqlError(
+					new Error("Internal error in Durable Object storage caused object to be reset"),
+				);
+			})
+			.mockReturnValueOnce("ok");
+
+		expect(runKernelSqlWithRetry(run)).toBe("ok");
+		expect(run).toHaveBeenCalledTimes(2);
+	});
+
+	it("rethrows a non-transient error without retrying", () => {
+		const error = new SqlError(new Error("no such column: bogus"));
+		const run = vi.fn<() => never>().mockImplementation(() => {
+			throw error;
+		});
+
+		expect(() => runKernelSqlWithRetry(run)).toThrow(error);
+		expect(run).toHaveBeenCalledTimes(1);
+	});
+
+	it("gives up after a bounded number of transient failures", () => {
+		const run = vi.fn<() => never>().mockImplementation(() => {
+			throw new SqlError(
+				new Error("Internal error in Durable Object storage caused object to be reset"),
+			);
+		});
+
+		expect(() => runKernelSqlWithRetry(run)).toThrow(/caused object to be reset/);
+		expect(run).toHaveBeenCalledTimes(3);
+	});
+});

src/features/workspaces/kernel/workspace-kernel-storage.ts

@@ -0,0 +1,74 @@
+/**
+ * Cloudflare can evict or reset a Durable Object mid-query — a transient
+ * platform event. When it happens `ctx.storage.sql.exec` throws
+ * "Internal error in Durable Object storage caused object to be reset", which
+ * the `agents` runtime rewraps as a `SqlError` ("SQL query failed: …"). These
+ * resets are not query defects, so we recognise them and retry rather than let
+ * them surface as unhandled failures in error tracking.
+ *
+ * `@cloudflare/sandbox`'s `isPlatformTransientError` covers the neighbouring
+ * cases (superseded isolates, lost connections, the *startup* storage reset),
+ * but not this mid-query reset variant — and importing it here would pull the
+ * worker-only sandbox runtime into unrelated code. So we match the transient
+ * signatures we care about directly.
+ */
+const TRANSIENT_STORAGE_PATTERNS = [
+	// Durable Object evicted/reset mid-query, including the startup variant.
+	/caused object to be reset/i,
+	// Isolate superseded by a code update.
+	/reset because its code was updated/i,
+	/this script has been upgraded/i,
+	// Storage connection dropped underneath the query.
+	/network connection lost/i,
+];
+
+const KERNEL_SQL_MAX_ATTEMPTS = 3;
+
+export function isTransientStorageError(error: unknown): boolean {
+	for (const candidate of errorAndCauses(error)) {
+		if (!(candidate instanceof Error)) {
+			continue;
+		}
+
+		if (TRANSIENT_STORAGE_PATTERNS.some((pattern) => pattern.test(candidate.message))) {
+			return true;
+		}
+	}
+
+	return false;
+}
+
+/**
+ * Runs a Durable Object SQL operation, retrying a bounded number of times when
+ * it fails with a transient storage reset. Storage ops are cheap and the reset
+ * is momentary, so an immediate retry usually succeeds. Non-transient errors
+ * (real query defects) are rethrown on the first attempt.
+ */
+export function runKernelSqlWithRetry<T>(run: () => T): T {
+	let lastError: unknown;
+
+	for (let attempt = 1; attempt <= KERNEL_SQL_MAX_ATTEMPTS; attempt += 1) {
+		try {
+			return run();
+		} catch (error) {
+			if (!isTransientStorageError(error)) {
+				throw error;
+			}
+
+			lastError = error;
+		}
+	}
+
+	throw lastError;
+}
+
+function* errorAndCauses(error: unknown): Iterable<unknown> {
+	const seen = new Set<unknown>();
+	let current: unknown = error;
+
+	while (current && !seen.has(current)) {
+		seen.add(current);
+		yield current;
+		current = current instanceof Error ? current.cause : undefined;
+	}
+}

src/features/workspaces/kernel/workspace-kernel.ts

@@ -16,6 +16,10 @@ import {
 	initializeWorkspaceKernelStorage,
 	type WorkspaceKernelSql,
 } from "#/features/workspaces/kernel/workspace-kernel-schema";
+import {
+	isTransientStorageError,
+	runKernelSqlWithRetry,
+} from "#/features/workspaces/kernel/workspace-kernel-storage";
 import { WorkspaceKernelRelations } from "#/features/workspaces/kernel/workspace-kernel-relations";
 import { WorkspaceKernelStore } from "#/features/workspaces/kernel/workspace-kernel-store";
 import type {
@@ -52,7 +56,7 @@ export { setWorkspaceKernelUserHeaders };
 
 export class WorkspaceKernel extends Agent<Cloudflare.Env> {
 	private readonly kernelSql: WorkspaceKernelSql = (strings, ...values) =>
-		this.sql(strings, ...values);
+		runKernelSqlWithRetry(() => this.sql(strings, ...values));
 	private readonly workspace = new ShellWorkspace({
 		sql: this.ctx.storage.sql,
 		r2: this.env.WORKSPACE_KERNEL_FILES,
@@ -219,15 +223,23 @@ export class WorkspaceKernel extends Agent<Cloudflare.Env> {
 			failure = error;
 			throw error;
 		} finally {
+			// A transient Durable Object reset that outlives the bounded SQL retry
+			// is infrastructure noise, not a broken flow — log it as a rejected
+			// outcome instead of capturing it as an unhandled exception.
+			const transientReset = failure !== undefined && isTransientStorageError(failure);
+
 			recordOperationalOutcome({
 				distinctId: input.actorUserId ?? undefined,
-				error: failure,
+				error: transientReset ? undefined : failure,
 				event: "workspace_mutation",
+				outcome: transientReset ? "rejected" : undefined,
 				fields: {
 					duration_ms: Date.now() - startedAt,
+					error_type: transientReset && failure instanceof Error ? failure.name : undefined,
 					operation,
 					operation_id: input.clientMutationId,
 					requested_count: requestedCount,
+					transient_storage_reset: transientReset ? true : undefined,
 					user_id: input.actorUserId,
 					workspace_id: this.name,
 				},