From 469afa99b9c5eddb38c5fa928f901b6af3c9e8c7 Mon Sep 17 00:00:00 2001 From: "posthog[bot]" <206114724+posthog[bot]@users.noreply.github.com> Date: Sat, 11 Jul 2026 21:42:58 +0000 Subject: [PATCH] fix(workspaces): retry transient Durable Object storage resets MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cloudflare can evict or reset a Durable Object mid-query, causing `ctx.storage.sql.exec` to throw "…caused object to be reset", which the `agents` runtime rewraps as a SqlError. The WorkspaceKernel had no transient classification, so these platform blips bubbled up and were captured to error tracking as unhandled failures. Classify these resets as transient, retry the SQL op a bounded number of times (storage ops are cheap and the reset is momentary), and log a surviving reset as a rejected outcome instead of an unhandled exception. Generated-By: PostHog Code Task-Id: 5357de1a-dc12-4c8f-ba43-329df7ac0183 --- .../kernel/workspace-kernel-storage.test.ts | 77 +++++++++++++++++++ .../kernel/workspace-kernel-storage.ts | 74 ++++++++++++++++++ .../workspaces/kernel/workspace-kernel.ts | 16 +++- 3 files changed, 165 insertions(+), 2 deletions(-) create mode 100644 src/features/workspaces/kernel/workspace-kernel-storage.test.ts create mode 100644 src/features/workspaces/kernel/workspace-kernel-storage.ts diff --git a/src/features/workspaces/kernel/workspace-kernel-storage.test.ts b/src/features/workspaces/kernel/workspace-kernel-storage.test.ts new file mode 100644 index 00000000..427c4f85 --- /dev/null +++ b/src/features/workspaces/kernel/workspace-kernel-storage.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, it, vi } from "vitest"; + +import { + isTransientStorageError, + runKernelSqlWithRetry, +} from "#/features/workspaces/kernel/workspace-kernel-storage"; + +class SqlError extends Error { + constructor(cause: unknown) { + const message = cause instanceof Error ? cause.message : String(cause); + super(`SQL query failed: ${message}`); + this.name = "SqlError"; + this.cause = cause; + } +} + +describe("workspace kernel storage transient errors", () => { + it("classifies a mid-query Durable Object reset as transient", () => { + const error = new SqlError( + new Error("Internal error in Durable Object storage caused object to be reset"), + ); + + expect(isTransientStorageError(error)).toBe(true); + }); + + it("classifies the underlying reset error directly", () => { + const error = new Error("Internal error in Durable Object storage caused object to be reset"); + + expect(isTransientStorageError(error)).toBe(true); + }); + + it("does not classify a genuine query defect as transient", () => { + const error = new SqlError(new Error("no such column: bogus")); + + expect(isTransientStorageError(error)).toBe(false); + }); + + it("tolerates non-error values", () => { + expect(isTransientStorageError("boom")).toBe(false); + expect(isTransientStorageError(undefined)).toBe(false); + }); + + it("retries a transient reset and returns once it succeeds", () => { + const run = vi + .fn<() => string>() + .mockImplementationOnce(() => { + throw new SqlError( + new Error("Internal error in Durable Object storage caused object to be reset"), + ); + }) + .mockReturnValueOnce("ok"); + + expect(runKernelSqlWithRetry(run)).toBe("ok"); + expect(run).toHaveBeenCalledTimes(2); + }); + + it("rethrows a non-transient error without retrying", () => { + const error = new SqlError(new Error("no such column: bogus")); + const run = vi.fn<() => never>().mockImplementation(() => { + throw error; + }); + + expect(() => runKernelSqlWithRetry(run)).toThrow(error); + expect(run).toHaveBeenCalledTimes(1); + }); + + it("gives up after a bounded number of transient failures", () => { + const run = vi.fn<() => never>().mockImplementation(() => { + throw new SqlError( + new Error("Internal error in Durable Object storage caused object to be reset"), + ); + }); + + expect(() => runKernelSqlWithRetry(run)).toThrow(/caused object to be reset/); + expect(run).toHaveBeenCalledTimes(3); + }); +}); diff --git a/src/features/workspaces/kernel/workspace-kernel-storage.ts b/src/features/workspaces/kernel/workspace-kernel-storage.ts new file mode 100644 index 00000000..2c64d978 --- /dev/null +++ b/src/features/workspaces/kernel/workspace-kernel-storage.ts @@ -0,0 +1,74 @@ +/** + * Cloudflare can evict or reset a Durable Object mid-query — a transient + * platform event. When it happens `ctx.storage.sql.exec` throws + * "Internal error in Durable Object storage caused object to be reset", which + * the `agents` runtime rewraps as a `SqlError` ("SQL query failed: …"). These + * resets are not query defects, so we recognise them and retry rather than let + * them surface as unhandled failures in error tracking. + * + * `@cloudflare/sandbox`'s `isPlatformTransientError` covers the neighbouring + * cases (superseded isolates, lost connections, the *startup* storage reset), + * but not this mid-query reset variant — and importing it here would pull the + * worker-only sandbox runtime into unrelated code. So we match the transient + * signatures we care about directly. + */ +const TRANSIENT_STORAGE_PATTERNS = [ + // Durable Object evicted/reset mid-query, including the startup variant. + /caused object to be reset/i, + // Isolate superseded by a code update. + /reset because its code was updated/i, + /this script has been upgraded/i, + // Storage connection dropped underneath the query. + /network connection lost/i, +]; + +const KERNEL_SQL_MAX_ATTEMPTS = 3; + +export function isTransientStorageError(error: unknown): boolean { + for (const candidate of errorAndCauses(error)) { + if (!(candidate instanceof Error)) { + continue; + } + + if (TRANSIENT_STORAGE_PATTERNS.some((pattern) => pattern.test(candidate.message))) { + return true; + } + } + + return false; +} + +/** + * Runs a Durable Object SQL operation, retrying a bounded number of times when + * it fails with a transient storage reset. Storage ops are cheap and the reset + * is momentary, so an immediate retry usually succeeds. Non-transient errors + * (real query defects) are rethrown on the first attempt. + */ +export function runKernelSqlWithRetry(run: () => T): T { + let lastError: unknown; + + for (let attempt = 1; attempt <= KERNEL_SQL_MAX_ATTEMPTS; attempt += 1) { + try { + return run(); + } catch (error) { + if (!isTransientStorageError(error)) { + throw error; + } + + lastError = error; + } + } + + throw lastError; +} + +function* errorAndCauses(error: unknown): Iterable { + const seen = new Set(); + let current: unknown = error; + + while (current && !seen.has(current)) { + seen.add(current); + yield current; + current = current instanceof Error ? current.cause : undefined; + } +} diff --git a/src/features/workspaces/kernel/workspace-kernel.ts b/src/features/workspaces/kernel/workspace-kernel.ts index 7dcc0f67..74db2df7 100644 --- a/src/features/workspaces/kernel/workspace-kernel.ts +++ b/src/features/workspaces/kernel/workspace-kernel.ts @@ -16,6 +16,10 @@ import { initializeWorkspaceKernelStorage, type WorkspaceKernelSql, } from "#/features/workspaces/kernel/workspace-kernel-schema"; +import { + isTransientStorageError, + runKernelSqlWithRetry, +} from "#/features/workspaces/kernel/workspace-kernel-storage"; import { WorkspaceKernelRelations } from "#/features/workspaces/kernel/workspace-kernel-relations"; import { WorkspaceKernelStore } from "#/features/workspaces/kernel/workspace-kernel-store"; import type { @@ -52,7 +56,7 @@ export { setWorkspaceKernelUserHeaders }; export class WorkspaceKernel extends Agent { private readonly kernelSql: WorkspaceKernelSql = (strings, ...values) => - this.sql(strings, ...values); + runKernelSqlWithRetry(() => this.sql(strings, ...values)); private readonly workspace = new ShellWorkspace({ sql: this.ctx.storage.sql, r2: this.env.WORKSPACE_KERNEL_FILES, @@ -219,15 +223,23 @@ export class WorkspaceKernel extends Agent { failure = error; throw error; } finally { + // A transient Durable Object reset that outlives the bounded SQL retry + // is infrastructure noise, not a broken flow — log it as a rejected + // outcome instead of capturing it as an unhandled exception. + const transientReset = failure !== undefined && isTransientStorageError(failure); + recordOperationalOutcome({ distinctId: input.actorUserId ?? undefined, - error: failure, + error: transientReset ? undefined : failure, event: "workspace_mutation", + outcome: transientReset ? "rejected" : undefined, fields: { duration_ms: Date.now() - startedAt, + error_type: transientReset && failure instanceof Error ? failure.name : undefined, operation, operation_id: input.clientMutationId, requested_count: requestedCount, + transient_storage_reset: transientReset ? true : undefined, user_id: input.actorUserId, workspace_id: this.name, },