diff --git a/CHANGELOG.md b/CHANGELOG.md index d2c3661..1aff099 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,15 @@ ## Unreleased - Updated maintenance dependencies and image baselines: workerd runtime images now use distroless `base-debian13`, local and Kubernetes Valkey use `valkey/valkey:9.1-alpine`, and root development tooling moved to current patch/minor releases. +- Consolidated duplicated Control request/error/retry paths, Redis locks and key grammars, queue and cron projections, S3 retry policy, observability helpers, and Rust sidecar contracts into canonical owners with shared behavioral and cross-language fixture coverage. +- Hardened the tenant/runtime boundary: gateway strips private `x-wdl-*` headers, generated wrappers and D1/R2/DO facades resist tenant prototype mutation, owner records and forwarding targets require service-specific private endpoints, ordinary nested DO calls cannot override the parent request id through request headers or public facade options, and DO ownership retries rely on private do-runtime markers without replaying unknown-outcome non-idempotent requests. +- Tightened forward-only input contracts: dynamic Workers reject explicit `compatibility_date` values earlier than `2026-04-01` and error-serialization flags that conflict with the effective date or WDL's required enhanced serialization, internal auth tokens must be visible ASCII without whitespace or commas, request ids are visible ASCII, secret names reject reserved `Object.prototype` keys, DO class names are capped so every shard fits the host-id limit, and `Headers`-form R2 metadata requires a canonical IMF-fixdate `Expires` value. +- Made required bundle metadata, queue base64 bodies, and persisted secret envelopes fail closed under their owning contracts; JavaScript and Rust now share the secret-envelope, queue-key, request-id, internal-auth, worker-version, scheduler-projection, and workflow-limit fixtures. +- Normalized `PLATFORM_DOMAIN` across routing tiers, stopped `/whoami` from advertising the internal `workers.local` fallback as a public namespace URL, and aligned Compose/Kubernetes/test wiring with the consolidated owners. +- Removed the unreachable DO inline worker-code test hook and its Terraform switch; do-runtime invoke paths now accept only canonical persisted bundle identities and reject non-canonical or oversized host ids. +- Aligned scheduler and Workflows Compose, Kubernetes, and ECS stop windows with their 25-second application drain so rollout and scale-in do not terminate in-flight work before the configured drain completes. +- Upgraded `@wdl-dev/aws-sigv4` to 3.0.1; WDL's S3-only URL-input paths preserve their existing signatures while adopting stable request snapshots, fail-closed redirects, lowercase region validation, and non-blocking response cleanup. +- Closed workflow restart/version-delete races, made malformed workflow and pattern projections fail closed, and ensured whole-worker delete removes orphan workflow definitions. ## wdl.20260701.1 - 2026-07-08 diff --git a/auth/index.js b/auth/index.js index 5d73e55..a9caaa3 100644 --- a/auth/index.js +++ b/auth/index.js @@ -37,6 +37,12 @@ import { tokenKey, } from "auth-runtime"; import { ROLES } from "shared-auth-roles"; +import { + acquireTokenLock, + createTokenLock, + releaseTokenLock, +} from "shared-redis-lock"; +import { NAMESPACES_KEY } from "shared-version"; const utf8Decoder = new TextDecoder(); @@ -128,47 +134,40 @@ async function scanTokenRecords(runtime, session) { /** * @param {import("shared-redis").RedisSession} session - * @param {string} key - * @param {string} token - */ -async function acquireDelegatedIssueLock(session, key, token) { - const reply = await session.multi() - .set(key, token, { nx: true, ttl: DELEGATED_ISSUE_LOCK_TTL_SECONDS }) - .exec(); - return Array.isArray(reply) && reply[0] === "OK"; -} - -/** - * @param {import("shared-redis").RedisSession} session - * @param {string} key - * @param {string} token + * @param {{ key: string, token: string }} lock * @param {ReturnType} runtime * @param {string | null} requestId */ -async function releaseDelegatedIssueLock(session, key, token, runtime, requestId) { - try { - await session.delIfEq(key, token); - } catch (err) { - runtime.logAuth("warn", "auth_delegated_issue_lock_release_failed", requestId, { +async function releaseDelegatedIssueLock(session, lock, runtime, requestId) { + await releaseTokenLock(session, lock, { + onError: (err) => runtime.logAuth("warn", "auth_delegated_issue_lock_release_failed", requestId, { ...formatError(err), - }); - // Do not turn a completed issue into an unknown/retryable outcome; the - // token-scoped lock has a short TTL and will self-clear. - } + }), + }); } /** * @param {import("shared-redis").RedisSession} session - * @param {string} key - * @param {string} token + * @param {{ key: string, token: string }} lock + * @param {ReturnType} runtime + * @param {string} issuerTokenId + * @param {string} templateId */ -async function watchDelegatedIssueLockHeld(session, key, token) { - await session.watch(key); - const current = await session.get(key); - if (current !== token) { +async function watchDelegatedIssuePrerequisites( + session, + lock, + runtime, + issuerTokenId, + templateId, +) { + await session.watch(lock.key, tokenKey(issuerTokenId)); + const current = await session.get(lock.key); + if (current !== lock.token) { throw new AuthPolicyError(409, "delegated_issue_busy", "delegated issue lock expired before the token could be issued; retry"); } + const issuer = await runtime.readRecord(session, issuerTokenId); + validateIssuerForDelegatedIssue(issuer, templateId); } /** @param {number} lockAttemptStartedAtMs */ @@ -225,7 +224,7 @@ async function chooseDelegatedNamespace(session, records, template) { for (let attempt = 0; attempt < DELEGATED_ISSUE_NAMESPACE_RETRIES; attempt += 1) { const ns = template.nsGenerator.prefix + randomHex(template.nsGenerator.randomHexBytes); if (!isValidTenantNs(ns)) continue; - if (await session.sIsMember("namespaces", ns)) continue; + if (await session.sIsMember(NAMESPACES_KEY, ns)) continue; if (tokenNamespaceClaimExists(records, ns)) continue; const label = renderDelegatedIssueLabel(template, ns); return { ns, label }; @@ -486,19 +485,25 @@ export default class Auth extends WorkerEntrypoint { } const lockKey = delegatedIssueLockKey(issuerTokenId, templateId); - const lockToken = randomHex(16); + const issueLock = createTokenLock(lockKey); const lockAttemptStartedAtMs = Date.now(); + let locked; try { - const locked = await acquireDelegatedIssueLock(session, lockKey, lockToken); - if (!locked) { - throw new AuthPolicyError(409, "delegated_issue_busy", - "delegated issue is already in progress"); - } + locked = await acquireTokenLock(session, issueLock, { + ttlSeconds: DELEGATED_ISSUE_LOCK_TTL_SECONDS, + }); } catch (err) { - await releaseDelegatedIssueLock(session, lockKey, lockToken, runtime, requestId); + // A failed SET reply has an unknown server-side outcome and leaves this + // RESP session untrustworthy. The lock TTL is the recovery path. return runtime.rethrowPolicy(requestId, "delegated_issue", err); } + if (!locked) { + return runtime.rethrowPolicy(requestId, "delegated_issue", + new AuthPolicyError(409, "delegated_issue_busy", + "delegated issue is already in progress")); + } + let canReleaseIssueLock = true; try { const nowMs = Date.now(); const tokenRecords = await scanTokenRecords(runtime, session); @@ -513,7 +518,13 @@ export default class Auth extends WorkerEntrypoint { } const namespaceChoice = await chooseDelegatedNamespace(session, tokenRecords, template); assertDelegatedIssueLockLocalBudget(lockAttemptStartedAtMs); - await watchDelegatedIssueLockHeld(session, lockKey, lockToken); + await watchDelegatedIssuePrerequisites( + session, + issueLock, + runtime, + issuerTokenId, + templateId, + ); const tokenId = generateTokenId(); const { ns, label } = namespaceChoice; const expiresAt = new Date(nowMs + template.ttlSeconds * 1000).toISOString(); @@ -542,6 +553,10 @@ export default class Auth extends WorkerEntrypoint { throw new AuthPolicyError(409, "delegated_issue_busy", "delegated issue lock changed before the token could be issued; retry"); } + // Any other EXEC failure has an unknown transaction outcome. Keep + // the lock and let its TTL bound recovery rather than retrying an + // issuance that may already have committed. + canReleaseIssueLock = false; throw err; } runtime.recordLifecycleOk("delegated_issue", requestId, { @@ -565,7 +580,9 @@ export default class Auth extends WorkerEntrypoint { } catch (err) { return runtime.rethrowPolicy(requestId, "delegated_issue", err); } finally { - await releaseDelegatedIssueLock(session, lockKey, lockToken, runtime, requestId); + if (canReleaseIssueLock) { + await releaseDelegatedIssueLock(session, issueLock, runtime, requestId); + } } }); } diff --git a/auth/lib.js b/auth/lib.js index c00d31e..5e48a7e 100644 --- a/auth/lib.js +++ b/auth/lib.js @@ -53,9 +53,9 @@ export { randomHex }; * }} TokenRecord */ -// Mirrors control/routing.js#RoutingError shape — index.js maps thrown -// instances to HTTP code so policy rejections stay distinct from Redis / -// code exceptions (which 503 fail-closed). +// Mirrors the control/routing status + machine-code shape on purpose, but stays +// auth-local: enhanced_error_serialization preserves {status, reason}, and +// control maps that distinct policy contract without importing auth internals. export class AuthPolicyError extends Error { /** * @param {number} status diff --git a/auth/runtime.js b/auth/runtime.js index 0fe10e8..d71ce21 100644 --- a/auth/runtime.js +++ b/auth/runtime.js @@ -19,6 +19,7 @@ import { import { actionCategory, } from "shared-auth-roles"; +import { withOptimisticRetries } from "shared-optimistic-retry"; /** * @typedef {import("shared-redis").RedisSession} RedisSession @@ -256,28 +257,31 @@ async function ensureBootstrapHash(redis, requestId, logAuth, desiredHash) { return true; }; - for (let attempt = 0; attempt < 5; attempt++) { - try { + await withOptimisticRetries( + async () => { if (typeof redis.session === "function") await redis.session(rotateBootstrap); else await rotateBootstrap(/** @type {RedisAuthSession} */ (redis)); - bootstrapMissingLogged = false; - verifyBootstrapHash = desiredHash; - logAuth("info", "auth_bootstrap", requestId, { - outcome: "ok", - token_id: BOOTSTRAP_TOKEN_ID, - rotated: rotatedFromExisting, - }); - return; - } catch (err) { - if (err instanceof WatchError) continue; - throw err; + return true; + }, + { + attempts: 5, + isRetryableError: (err) => err instanceof WatchError, + onExhausted: () => { + logAuth("error", "auth_bootstrap", requestId, { + outcome: "error", + reason: "bootstrap_watch_exhausted", + }); + throw new Error("bootstrap upsert WATCH retry exhausted"); + }, } - } - logAuth("error", "auth_bootstrap", requestId, { - outcome: "error", - reason: "bootstrap_watch_exhausted", + ); + bootstrapMissingLogged = false; + verifyBootstrapHash = desiredHash; + logAuth("info", "auth_bootstrap", requestId, { + outcome: "ok", + token_id: BOOTSTRAP_TOKEN_ID, + rotated: rotatedFromExisting, }); - throw new Error("bootstrap upsert WATCH retry exhausted"); } /** @param {AuthLogger} logAuth @param {string | null} requestId @param {string} command @param {unknown} err @returns {never} */ diff --git a/control/bindings.js b/control/bindings.js index 9f3416d..fc34ad3 100644 --- a/control/bindings.js +++ b/control/bindings.js @@ -7,11 +7,13 @@ import { QUEUE_NAME_RE, BINDING_NAME_RE, KV_ID_RE, + D1_DATABASE_ID_RE, R2_BUCKET_NAME_RE, WDL_RESERVED_BINDING_RE, RESERVED_OBJECT_KEYS, RESERVED_TENANT_NS, WDL_RESERVED_ENTRYPOINT_RE, + MAX_DO_CLASS_NAME_BYTES, isValidJsIdentifier, isValidJsClassDeclarationName, } from "shared-ns-pattern"; @@ -26,8 +28,6 @@ import { MAX_QUEUE_DELAY_SECONDS, } from "control-lib"; -const D1_DATABASE_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/; - // Narrower than SECRET_KEY_RE (no lowercase) so platform slot names read // as registered identifiers rather than local vars. export const PLATFORM_KEY_RE = /^[A-Z_][A-Z0-9_]*$/; @@ -143,9 +143,14 @@ const BINDING_VALIDATORS = Object.assign(Object.create(null), /** @type {Record< }, do(b, name) { const className = b.className; - if (typeof className !== "string" || !isValidJsClassDeclarationName(className)) { + if ( + typeof className !== "string" || + !isValidJsClassDeclarationName(className) || + className.length > MAX_DO_CLASS_NAME_BYTES + ) { throw new Error( - `bindings.${name}: do className must be a valid JS class declaration name, got ${JSON.stringify(className)}` + `bindings.${name}: do className must be a valid JS class declaration name of at most ` + + `${MAX_DO_CLASS_NAME_BYTES} bytes, got ${JSON.stringify(className)}` ); } assertNotRuntimeReservedEntrypoint(`bindings.${name}`, className); @@ -457,6 +462,9 @@ export async function linkServiceBinding({ targetMeta = await lookupTargetMeta(targetNs, service, targetVersion); if (!targetMeta) targetMeta = {}; } catch (err) { + // The injected reader may classify persisted target metadata separately + // from transport failures; preserve that domain error at the link boundary. + if (err instanceof LinkError) throw err; const message = errorMessage(err); throw new LinkError(502, "service_binding_target_meta_unavailable", `bindings.${bindingName}: failed to read target meta: ${message}`); diff --git a/control/bundle.js b/control/bundle.js index ed3ab19..a502e7e 100644 --- a/control/bundle.js +++ b/control/bundle.js @@ -12,8 +12,16 @@ import { isValidJsClassDeclarationName, validateModulePath, } from "shared-ns-pattern"; -import { firstWorkerdExperimentalCompatFlag } from "shared-workerd-compat-flags"; +import { + DEFAULT_DYNAMIC_WORKER_COMPATIBILITY_DATE, + ENHANCED_ERROR_SERIALIZATION_DEFAULT_DATE, + ENHANCED_ERROR_SERIALIZATION_FLAG, + LEGACY_ERROR_SERIALIZATION_FLAG, + MIN_DYNAMIC_WORKER_COMPATIBILITY_DATE, + firstWorkerdExperimentalCompatFlag, +} from "shared-workerd-compat-flags"; import { normalizeBindings, validateBindings } from "control-bindings"; +import { parseWorkerdDependencyVersion } from "control-lib"; import PACKAGE_JSON_SOURCE from "wdl-package-json-source"; export class BundleConfigError extends Error { @@ -26,22 +34,12 @@ export class BundleConfigError extends Error { } const COMPATIBILITY_DATE_RE = /^(\d{4})-(\d{2})-(\d{2})$/; -const WORKERD_VERSION_RE = /^1\.(\d{4})(\d{2})(\d{2})\.\d+$/; /** @param {string} source */ export function maxWorkerCompatibilityDateFromPackageJson(source) { - let parsed; - try { - parsed = JSON.parse(source); - } catch { - return null; - } - const raw = parsed?.dependencies?.workerd; - if (typeof raw !== "string") return null; - const version = raw.replace(/^[~^]/, ""); - const match = WORKERD_VERSION_RE.exec(version); - if (!match) return null; - const max = new Date(Date.UTC(Number(match[1]), Number(match[2]) - 1, Number(match[3]) + 7)); + const version = parseWorkerdDependencyVersion(source); + if (!version) return null; + const max = new Date(Date.UTC(version.year, version.month - 1, version.day + 7)); return utcDateString(max); } @@ -88,6 +86,11 @@ export function validateCompatibilityDate(value, today = new Date()) { ) { throw new Error(`compatibilityDate must be a real calendar date, got ${JSON.stringify(value)}`); } + if (value < MIN_DYNAMIC_WORKER_COMPATIBILITY_DATE) { + throw new Error( + `compatibilityDate ${value} is older than WDL supports (${MIN_DYNAMIC_WORKER_COMPATIBILITY_DATE})` + ); + } const todayUtc = utcDateString(today); if (value > todayUtc) { throw new Error(`compatibilityDate ${value} must not be later than today UTC (${todayUtc})`); @@ -150,8 +153,8 @@ export function normalizeModule(value) { // like { compatibilityFlags: "nodejs_compat" } or { flag: true } drops // the user's declaration and leaves only the platform floor — looks // healthy, isn't. -/** @param {unknown} flags */ -function validateCompatibilityFlags(flags) { +/** @param {unknown} flags @param {string} compatibilityDate */ +function validateCompatibilityFlags(flags, compatibilityDate) { if (flags === undefined || flags === null) return; if (!Array.isArray(flags)) { throw new Error( @@ -173,6 +176,23 @@ function validateCompatibilityFlags(flags) { `compatibilityFlags contains experimental workerd flag ${JSON.stringify(experimentalFlag)}, which WDL does not support for tenant workers` ); } + if (flags.includes(LEGACY_ERROR_SERIALIZATION_FLAG)) { + throw new BundleConfigError( + 400, + "compatibility_flag_unsupported", + `${JSON.stringify(LEGACY_ERROR_SERIALIZATION_FLAG)} is not supported because WDL requires enhanced error serialization` + ); + } + if ( + compatibilityDate >= ENHANCED_ERROR_SERIALIZATION_DEFAULT_DATE && + flags.includes(ENHANCED_ERROR_SERIALIZATION_FLAG) + ) { + throw new BundleConfigError( + 400, + "compatibility_flag_unsupported", + `${JSON.stringify(ENHANCED_ERROR_SERIALIZATION_FLAG)} became the workerd default on ${ENHANCED_ERROR_SERIALIZATION_DEFAULT_DATE} and must not be specified for compatibilityDate ${compatibilityDate}` + ); + } } /** @param {unknown} workflows */ @@ -296,7 +316,10 @@ export function prepareBundle(mainModule, rawModules, extras = {}) { } validateBindings(extras.bindings); const compatibilityDate = validateCompatibilityDate(extras.compatibilityDate); - validateCompatibilityFlags(extras.compatibilityFlags); + validateCompatibilityFlags( + extras.compatibilityFlags, + compatibilityDate ?? DEFAULT_DYNAMIC_WORKER_COMPATIBILITY_DATE + ); const vars = normalizeVars(extras.vars); // Null-proto: any path slipping past validateModulePath can't take // the __proto__ setter path on assignment. diff --git a/control/d1-migrations.js b/control/d1-migrations.js index 144cc47..958c88f 100644 --- a/control/d1-migrations.js +++ b/control/d1-migrations.js @@ -3,10 +3,16 @@ import { jsonError, readJsonBody, errMessage, - randomHex, + formatError, requireControlLog, requireControlRedis, } from "control-shared"; +import { + acquireTokenLock, + createTokenLock, + releaseTokenLock, + renewTokenLock, +} from "shared-redis-lock"; import { migrationStatus, MIGRATIONS_TABLE_SQL, @@ -60,28 +66,35 @@ function migrationProgress(applied, skipped) { async function acquireMigrationLock(ns, databaseId) { const redis = requireControlRedis(); const key = migrationLockKey(ns, databaseId); - const token = randomHex(); - const ok = await redis.set(key, token, { ttl: MIGRATION_LOCK_TTL_SECONDS, nx: true }); - if (!ok) return null; - return { key, token }; + const lock = createTokenLock(key); + return await acquireTokenLock(redis, lock, { ttlSeconds: MIGRATION_LOCK_TTL_SECONDS }) + ? lock + : null; } /** @param {MigrationLock | null} lock */ export async function renewMigrationLock(lock) { if (!lock) return { ok: false, reason: "lost" }; const redis = requireControlRedis(); - const renewed = await redis.set(lock.key, lock.token, { - ttl: MIGRATION_LOCK_TTL_SECONDS, - ifeq: lock.token, - }); + const renewed = await renewTokenLock(redis, lock, MIGRATION_LOCK_TTL_SECONDS); return renewed ? { ok: true } : { ok: false, reason: "lost" }; } -/** @param {MigrationLock | null} lock */ -async function releaseMigrationLock(lock) { +/** + * @param {MigrationLock | null} lock + * @param {{ log: import("control-shared").ControlLogger, ns: string, databaseId: string, requestId: string }} context + */ +async function releaseMigrationLock(lock, { log, ns, databaseId, requestId }) { if (!lock) return; const redis = requireControlRedis(); - await redis.delIfEq(lock.key, lock.token); + await releaseTokenLock(redis, lock, { + onError: (err) => log("warn", "d1_migration_lock_release_failed", { + request_id: requestId, + namespace: ns, + database_id: databaseId, + ...formatError(err), + }), + }); } /** @@ -320,7 +333,12 @@ export async function applyMigrations({ request, env, ns, databaseId, requestId appliedById.set(migration.id, record); } } finally { - await releaseMigrationLock(lock); + await releaseMigrationLock(lock, { + log, + ns, + databaseId: database.databaseId, + requestId, + }); } log("info", "d1_migrations_applied", { diff --git a/control/d1-model.js b/control/d1-model.js index 1c83c61..e570c10 100644 --- a/control/d1-model.js +++ b/control/d1-model.js @@ -1,7 +1,8 @@ import { bytesToHex } from "shared-hex"; +import { D1_DATABASE_ID_RE } from "shared-ns-pattern"; export { splitSqlStatements } from "shared-sql-splitter"; -export const D1_DATABASE_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/; +export { D1_DATABASE_ID_RE }; export const D1_DATABASE_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/; export const D1_MIGRATION_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,191}$/; export const EXECUTE_MODES = new Set(["all", "raw", "run", "exec"]); diff --git a/control/d1-runtime-client.js b/control/d1-runtime-client.js index c931539..0ff5d88 100644 --- a/control/d1-runtime-client.js +++ b/control/d1-runtime-client.js @@ -13,7 +13,7 @@ import { } from "shared-d1-query-wire"; import { sanitizeJsonErrorDetails } from "shared-respond"; import { withInternalAuth } from "shared-internal-auth"; -import { validOwnerEndpointForService } from "runtime-owner-endpoint"; +import { validOwnerEndpointForService } from "shared-owner-endpoint"; /** * @typedef {{ fetch(input: RequestInfo | URL, init?: RequestInit): Promise }} Fetcher @@ -33,7 +33,7 @@ function d1OwnerGenerationFromHeaders(headers) { const raw = headers.get("x-wdl-d1-owner-generation"); if (raw == null || raw === "") return null; const generation = Number(raw); - return Number.isInteger(generation) && generation >= 0 ? generation : null; + return Number.isSafeInteger(generation) && generation > 0 ? generation : null; } /** diff --git a/control/env-budget.js b/control/env-budget.js index 4ef3819..5f11e13 100644 --- a/control/env-budget.js +++ b/control/env-budget.js @@ -1,19 +1,37 @@ import { decryptSecretValue } from "shared-secret-envelope"; -import { errorMessage } from "shared-errors"; +import { BundleMetaError, parseBundleMeta } from "control-lib"; +import { buildWorkerEnv } from "runtime-load-env-build"; import { bundleKey } from "shared-version"; import { WatchError } from "shared-redis"; -const DO_BACKEND_BINDING = "__WDL_DO_BACKEND__"; -const DO_OWNER_NETWORK_BINDING = "__WDL_DO_OWNER_NETWORK__"; +export { BundleMetaError }; + const DO_ALARMS_BINDING = "__WDL_DO_ALARMS__"; -const WORKFLOWS_BACKEND_BINDING = "__WDL_WORKFLOWS_BACKEND__"; const ESTIMATED_ASSETS_CDN_BASE = "https://assets.invalid"; // Pessimistic placeholder for version strings that will be allocated after a // secret mutation. Redis INCR results are parsed as JS numbers today, so this // uses the longest safe integer-shaped `v` tag. export const WORKER_LOADER_ENV_VERSION_PLACEHOLDER = "v9007199254740991"; -const ESTIMATED_DO_STORAGE_ID = "do_00000000000000000000000000000000"; -const ESTIMATED_WORKFLOW_KEY = "wf_00000000000000000000000000000000"; +const ESTIMATED_DO_BACKEND = Object.freeze({ __wdlBinding: "internal", name: "DO_BACKEND" }); +const ESTIMATED_DO_OWNER_NETWORK = Object.freeze({ + __wdlBinding: "internal", + name: "DO_OWNER_NETWORK", +}); +const ESTIMATED_WORKFLOWS_BACKEND = Object.freeze({ + __wdlBinding: "internal", + name: "WORKFLOWS_BACKEND", +}); +/** @type {Parameters[7]} */ +const ESTIMATED_RUNTIME_CONTEXT = Object.freeze({ + exports: Object.freeze({ + KV: (/** @type {{ props: Record }} */ { props }) => ({ __wdlBinding: "kv", props }), + Assets: (/** @type {{ props: Record }} */ { props }) => ({ __wdlBinding: "assets", props }), + QueueProducer: (/** @type {{ props: Record }} */ { props }) => ({ __wdlBinding: "queue", props }), + D1Database: (/** @type {{ props: Record }} */ { props }) => ({ __wdlBinding: "d1", props }), + R2Bucket: (/** @type {{ props: Record }} */ { props }) => ({ __wdlBinding: "r2", props }), + ServiceBinding: (/** @type {{ props: Record }} */ { props }) => ({ __wdlBinding: "service", props }), + }), +}); // Mirrors workerd v1.20260701.1 // src/workerd/api/worker-loader.c++ MAX_DYNAMIC_WORKER_ENV_SIZE. @@ -37,6 +55,17 @@ export class WorkerEnvBudgetError extends Error { } } +/** @param {string} ns @param {string} worker @param {string} version @param {unknown} cause */ +function bundleMaterializationError(ns, worker, version, cause) { + return new BundleMetaError({ + namespace: ns, + worker, + version, + message: `Corrupt __meta__ for ${ns}/${worker}/${version}`, + cause, + }); +} + /** @param {Record | null | undefined} source */ function stringRecord(source) { /** @type {Record} */ @@ -54,126 +83,21 @@ function objectRecord(value) { : null; } -/** @param {unknown} value */ -function stringOrFallback(value, fallback = "") { - return typeof value === "string" ? value : fallback; -} - -/** - * @param {{ - * requiredCallerSecrets?: unknown, - * nsSecrets: Record, - * workerSecrets: Record, - * }} args - */ -function callerSecretsForBinding({ requiredCallerSecrets, nsSecrets, workerSecrets }) { - if (!Array.isArray(requiredCallerSecrets) || requiredCallerSecrets.length === 0) return undefined; - /** @type {Record} */ - const callerSecrets = Object.create(null); - for (const key of requiredCallerSecrets) { - if (typeof key !== "string") continue; - if (Object.hasOwn(workerSecrets, key)) { - callerSecrets[key] = workerSecrets[key]; - } else if (Object.hasOwn(nsSecrets, key)) { - callerSecrets[key] = nsSecrets[key]; - } - } - return callerSecrets; -} - -/** - * @param {{ - * name: string, - * spec: Record, - * meta: Record, - * ns: string, - * worker: string, - * version: string, - * assetsCdnBase: string, - * nsSecrets: Record, - * workerSecrets: Record, - * }} args - */ -function estimatedBindingEnvValue({ name, spec, meta, ns, worker, version, assetsCdnBase, nsSecrets, workerSecrets }) { - switch (spec.type) { - case "kv": - return { - __wdlBinding: "kv", - props: { ns, id: stringOrFallback(spec.id) }, - }; - case "assets": - return { - __wdlBinding: "assets", - props: { - cdnBase: assetsCdnBase, - prefix: stringOrFallback(objectRecord(meta.assets)?.prefix), - }, - }; - case "queue": - return { - __wdlBinding: "queue", - props: { - ns, - id: stringOrFallback(spec.id), - deliveryDelaySeconds: spec.deliveryDelaySeconds ?? 0, - }, - }; - case "d1": - return { - __wdlBinding: "d1", - props: { - ns, - databaseId: stringOrFallback(spec.databaseId), - binding: name, - }, - }; - case "r2": - return { - __wdlBinding: "r2", - props: { - ns, - bucketName: stringOrFallback(spec.bucketName), - binding: name, - }, - }; - case "do": { - const props = { - ns, - worker, - version, - doStorageId: stringOrFallback(spec.doStorageId, ESTIMATED_DO_STORAGE_ID), - binding: name, - className: stringOrFallback(spec.className), - }; - // DO bindings intentionally mirror runtime's default/factory output shape: - // props are top-level fields, while hostProxy carries the JSRPC namespace. - return { - __wdlBinding: "do", - ...props, - hostProxy: { __wdlBinding: "do-host-proxy", props }, - }; - } - case "service": { - const callerSecrets = callerSecretsForBinding({ - requiredCallerSecrets: spec.requiredCallerSecrets, - nsSecrets, - workerSecrets, - }); - return { - __wdlBinding: "service", - props: { - targetNs: stringOrFallback(spec.ns, ns), - targetWorker: stringOrFallback(spec.service), - targetVersion: stringOrFallback(spec.version), - targetEntrypoint: typeof spec.entrypoint === "string" ? spec.entrypoint : null, - callerNs: ns, - ...(callerSecrets ? { callerSecrets } : {}), - }, - }; - } - default: - return { __wdlBinding: stringOrFallback(spec.type, "unknown") }; - } +/** @param {{ name: string, spec: Record, ns: string, worker: string, version: string }} input */ +function estimatedDoBinding({ name, spec, ns, worker, version }) { + const props = { + ns, + worker, + version, + doStorageId: spec.doStorageId, + binding: name, + className: spec.className, + }; + return { + __wdlBinding: "do", + ...props, + hostProxy: { __wdlBinding: "do-host-proxy", props }, + }; } /** @@ -209,63 +133,37 @@ export function estimatedWorkerLoaderEnv({ const metaRecord = objectRecord(meta); if (!metaRecord || !worker) return env; - let hasDoBinding = false; - let doAlarmStorageId = ESTIMATED_DO_STORAGE_ID; - let hasWorkflowBinding = false; - const workflows = Array.isArray(metaRecord.workflows) ? metaRecord.workflows : []; - for (const workflow of workflows) { - const record = objectRecord(workflow); - if (!record) continue; - const binding = stringOrFallback(record.binding); - if (!binding) continue; - hasWorkflowBinding = true; - env[binding] = { - ns, - worker, - version, - name: stringOrFallback(record.name), - binding, - className: stringOrFallback(record.className), - workflowKey: stringOrFallback(record.workflowKey, ESTIMATED_WORKFLOW_KEY), - }; - } - - const bindings = objectRecord(metaRecord.bindings); - if (bindings) { - for (const [name, rawSpec] of Object.entries(bindings)) { - const spec = objectRecord(rawSpec); - if (!spec) continue; - env[name] = estimatedBindingEnvValue({ - name, - spec, - meta: metaRecord, - ns, - worker, - version, - assetsCdnBase: typeof assetsCdnBase === "string" && assetsCdnBase - ? assetsCdnBase - : ESTIMATED_ASSETS_CDN_BASE, - nsSecrets: nsSecretStrings, - workerSecrets: workerSecretStrings, - }); - if (spec.type === "do") { - hasDoBinding = true; - doAlarmStorageId = stringOrFallback(spec.doStorageId, ESTIMATED_DO_STORAGE_ID); - } + const estimated = buildWorkerEnv( + { ...metaRecord, vars: stringRecord(vars) }, + nsSecretStrings, + workerSecretStrings, + ns, + worker, + version, + typeof assetsCdnBase === "string" && assetsCdnBase + ? assetsCdnBase + : ESTIMATED_ASSETS_CDN_BASE, + ESTIMATED_RUNTIME_CONTEXT, + ESTIMATED_DO_BACKEND, + { + doOwnerNetwork: ESTIMATED_DO_OWNER_NETWORK, + doBindingFactory: estimatedDoBinding, + workflowsBackend: ESTIMATED_WORKFLOWS_BACKEND, } + ); + + let doAlarmStorageId = null; + for (const rawSpec of Object.values(objectRecord(metaRecord.bindings) || {})) { + const spec = objectRecord(rawSpec); + if (spec?.type === "do") doAlarmStorageId = spec.doStorageId; } - if (hasDoBinding) { - env[DO_BACKEND_BINDING] = { __wdlBinding: "internal", name: "DO_BACKEND" }; - env[DO_OWNER_NETWORK_BINDING] = { __wdlBinding: "internal", name: "DO_OWNER_NETWORK" }; - env[DO_ALARMS_BINDING] = { + if (typeof doAlarmStorageId === "string" && doAlarmStorageId) { + estimated[DO_ALARMS_BINDING] = { __wdlBinding: "do-alarms", props: { ns, worker, version, doStorageId: doAlarmStorageId }, }; } - if (hasWorkflowBinding) { - env[WORKFLOWS_BACKEND_BINDING] = { __wdlBinding: "internal", name: "WORKFLOWS_BACKEND" }; - } - return env; + return estimated; } /** @param {string} value */ @@ -471,39 +369,29 @@ export async function assertWorkerVersionsUserEnvBudget({ // whose command protocol is single-flight even though secret decryption is not. for (const { sourceVersion, estimatedVersion } of uniqueChecks) { const rawMeta = await redis.hGet(bundleKey(ns, worker, sourceVersion), "__meta__"); - if (typeof rawMeta !== "string") { - if (retryMissingVersions) throw new WatchError(); - throw new Error(`bundle metadata missing for ${ns}/${worker}@${sourceVersion}`); - } - /** @type {unknown} */ - let parsed; - try { - parsed = JSON.parse(rawMeta); - } catch (err) { - throw new Error( - `invalid bundle metadata for ${ns}/${worker}@${sourceVersion}: ${errorMessage(err)}`, - { cause: err } - ); - } - if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { - throw new Error( - `invalid bundle metadata for ${ns}/${worker}@${sourceVersion}: ` + - "__meta__ must be a JSON object" - ); - } - const meta = /** @type {Record} */ (parsed); - assertWorkerLoaderUserEnvBudget({ + if (typeof rawMeta !== "string" && retryMissingVersions) throw new WatchError(); + const meta = parseBundleMeta(rawMeta, { ns, worker, - version: estimatedVersion, - sourceVersion, - vars: meta.vars && typeof meta.vars === "object" && !Array.isArray(meta.vars) - ? /** @type {Record} */ (meta.vars) - : null, - nsSecrets, - workerSecrets, - meta, - assetsCdnBase, + version: sourceVersion, }); + try { + assertWorkerLoaderUserEnvBudget({ + ns, + worker, + version: estimatedVersion, + sourceVersion, + vars: meta.vars && typeof meta.vars === "object" && !Array.isArray(meta.vars) + ? /** @type {Record} */ (meta.vars) + : null, + nsSecrets, + workerSecrets, + meta, + assetsCdnBase, + }); + } catch (err) { + if (err instanceof WorkerEnvBudgetError || err instanceof BundleMetaError) throw err; + throw bundleMaterializationError(ns, worker, sourceVersion, err); + } } } diff --git a/control/errors.js b/control/errors.js new file mode 100644 index 0000000..12d1ba3 --- /dev/null +++ b/control/errors.js @@ -0,0 +1,185 @@ +import { jsonError } from "shared-respond"; +import { errorMessage } from "shared-errors"; + +// Control owns this base abort contract. Routing and auth remain separate; +// deploy subclasses it where commit cleanup needs its own catch boundary. + +/** @param {unknown} value @returns {value is Record} */ +function isRecord(value) { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +const SERVER_DIAGNOSTIC_DETAIL_KEYS = new Set([ + "cause", + "detail", + "error_detail", + "stack", + "stage", +]); +const CODED_ERROR_LOG_CONTEXT_RESERVED_KEYS = new Set([ + ...SERVER_DIAGNOSTIC_DETAIL_KEYS, + "code", + "error_message", + "message", + "metadata_version", + "reason", + "status", + "version", +]); +const CONTROL_LOG_STRING_MAX_CHARS = 2048; +const CONTROL_LOG_TRUNCATION_SUFFIX = "..."; + +/** @param {string} value */ +function boundedControlLogString(value) { + if (value.length <= CONTROL_LOG_STRING_MAX_CHARS) return value; + const prefixLength = CONTROL_LOG_STRING_MAX_CHARS - CONTROL_LOG_TRUNCATION_SUFFIX.length; + return `${value.slice(0, prefixLength)}${CONTROL_LOG_TRUNCATION_SUFFIX}`; +} + +/** @param {unknown} value */ +function codedErrorLogContext(value) { + if (!isRecord(value)) return {}; + return Object.fromEntries( + Object.entries(value).filter(([key]) => !CODED_ERROR_LOG_CONTEXT_RESERVED_KEYS.has(key)) + ); +} + +/** + * @param {number} status + * @param {Record} details + */ +function publicErrorDetails(status, details) { + if (status < 500) return details; + return Object.fromEntries( + Object.entries(details).filter(([key]) => !SERVER_DIAGNOSTIC_DETAIL_KEYS.has(key)) + ); +} + +export class ControlAbort extends Error { + /** + * @param {number} status + * @param {string} code + * @param {{ message?: string, [key: string]: unknown }} [details] + */ + constructor(status, code, details = {}) { + super(details?.message || code); + this.status = status; + this.code = code; + this.details = details; + } +} + +/** + * Return the bounded diagnostics that a server-side coded error may add to a + * structured log after those fields have been removed from the wire response. + * + * @param {ControlAbort} err + * @returns {Record} + */ +export function controlAbortLogDetails(err) { + if (err.status < 500 || !isRecord(err.details)) return {}; + /** @type {Record} */ + const out = {}; + if (typeof err.details.version === "string") { + out.metadata_version = boundedControlLogString(err.details.version); + } + if (typeof err.details.stage === "string") { + out.stage = boundedControlLogString(err.details.stage); + } + const detail = typeof err.details.error_detail === "string" + ? err.details.error_detail + : err.details.detail; + if (typeof detail === "string") { + out.error_detail = boundedControlLogString(detail); + } + return out; +} + +/** + * @param {unknown} err + * @param {string} [fallbackCode] + * @param {{ errorDetail?: string, context?: Record }} [diagnostics] + * @returns {{ status: number, reason: string, error_message: string, [key: string]: unknown }} + */ +export function codedErrorLogFields(err, fallbackCode = "internal_error", diagnostics = {}) { + const record = isRecord(err) ? err : {}; + const status = typeof record.status === "number" ? record.status : 500; + const reason = typeof record.code === "string" && record.code ? record.code : fallbackCode; + const message = typeof record.message === "string" ? record.message : errorMessage(err); + return { + ...codedErrorLogContext(diagnostics.context), + status, + reason: boundedControlLogString(reason), + error_message: boundedControlLogString(message), + ...(err instanceof ControlAbort ? controlAbortLogDetails(err) : {}), + ...(typeof diagnostics.errorDetail === "string" + ? { error_detail: boundedControlLogString(diagnostics.errorDetail) } + : {}), + }; +} + +// Domain errors flow through jsonError() so details cannot shadow the +// top-level wire contract. +/** + * @param {unknown} err + * @param {string} [fallbackCode] + * @param {Record} [extraDetails] + */ +export function codedErrorResponse(err, fallbackCode = "internal_error", extraDetails = {}) { + const record = isRecord(err) ? err : {}; + const details = isRecord(record.details) ? record.details : {}; + const status = typeof record.status === "number" ? record.status : 500; + const code = typeof record.code === "string" && record.code ? record.code : fallbackCode; + const recordMessage = typeof record.message === "string" && record.message ? record.message : undefined; + let message = "Internal error"; + if (status < 500) { + message = err instanceof ControlAbort + ? (typeof err.details.message === "string" && err.details.message) || err.message || code + : recordMessage || (typeof details.message === "string" && details.message) || code; + } + return jsonError( + status, + code, + message, + publicErrorDetails(status, { ...details, ...extraDetails }), + ); +} + +/** + * Log a secret-envelope provider failure internally while returning only the + * stable coded 503 contract to the caller. + * + * @param {{ + * err: import("shared-secret-envelope").SecretEnvelopeError, + * log: (level: string, event: string, fields: Record) => void, + * event: string, + * fields: Record, + * responseDetails?: Record, + * }} args + */ +export function secretEnvelopeErrorResponse({ + err, + log, + event, + fields, + responseDetails = {}, +}) { + const diagnostic = errorMessage(err); + log("error", event, { + ...fields, + ...codedErrorLogFields( + { status: 503, code: err.code, message: diagnostic }, + err.code, + { errorDetail: diagnostic } + ), + }); + return codedErrorResponse({ status: 503, code: err.code }, err.code, responseDetails); +} + +/** + * @param {ControlAbort} err + * @param {Record} [extraDetails] + */ +export function controlAbortResponse(err, extraDetails = {}) { + return codedErrorResponse(err, err.code, extraDetails); +} diff --git a/control/handlers/delete-plan.js b/control/handlers/delete-plan.js index bcb970f..183aca5 100644 --- a/control/handlers/delete-plan.js +++ b/control/handlers/delete-plan.js @@ -12,7 +12,7 @@ import { stageWorkerHidden, stageWorkerVersionIndexRemove, } from "control-lifecycle-indexes"; -import { bundleKey, patternsKey, routesKey } from "shared-version"; +import { NAMESPACES_KEY, bundleKey, nsHostsKey, patternsKey, routesKey } from "shared-version"; import { workerSecretsKey } from "shared-secret-keys"; /** @@ -35,6 +35,7 @@ import { workerSecretsKey } from "shared-secret-keys"; * hostsLosingNsOwnership: string[], * namespaceStillActive: boolean, * hasWorkerSecrets: boolean, + * hasWorkflowDefs: boolean, * }} DeleteInputs * @typedef {import("control-lifecycle-indexes").RedisMulti} RedisMulti * @typedef {{ routes: string, routesFlush: string, patterns: string }} DeleteChannels @@ -88,10 +89,10 @@ export function stageWorkerDelete(multi, { collected, channels }) { multi.hDel(routesKey(ns), name); } if (collected.hostsLosingNsOwnership.length) { - multi.sRem(`ns-hosts:${ns}`, collected.hostsLosingNsOwnership); + multi.sRem(nsHostsKey(ns), collected.hostsLosingNsOwnership); } if (!collected.namespaceStillActive) { - multi.sRem("namespaces", ns); + multi.sRem(NAMESPACES_KEY, ns); } multi.del(cronWorkerKey(ns, name)); diff --git a/control/handlers/delete.js b/control/handlers/delete.js index dceb9e0..e0ff591 100644 --- a/control/handlers/delete.js +++ b/control/handlers/delete.js @@ -7,22 +7,32 @@ import { jsonResponse, jsonError, requireControlLog, requireControlRedis, errMessage, - acquireDeleteLock, releaseDeleteLock, + acquireDeleteLock, releaseDeleteLock, renewDeleteLock, assertWorkflowDeleteAllowed, cleanupDoAlarmsForWorker, buildS3CleanupTaskId, recordCleanupIntentOrWarn, - ControlAbort, controlAbortResponse, + ControlAbort, codedErrorLogFields, controlAbortResponse, + withOptimisticRetries, ROUTES_CHANNEL, ROUTES_FLUSH_CHANNEL, PATTERNS_CHANNEL, } from "control-shared"; import { workerVersionsKey, referrersKey, - deleteLockKey, doObjectRegistryKey, doOwnerScopeScanPatternForStorage, doStorageIdKey, + workflowDefsKey, extractD1Refs, extractOutgoingRefs, formatReferrerBlocker, + bundleAssetPrefix, + parseBundleMeta, } from "control-lib"; -import { bundleKey, patternsKey, routesKey } from "shared-version"; +import { + WHOLE_DELETE_LOCK_KIND, + bundleKey, + deleteLockKey, + hostsKey, + patternsKey, + routesKey, +} from "shared-version"; import { workerSecretsKey } from "shared-secret-keys"; import { decodeBulk, WatchError } from "shared-redis"; import { queueConsumerScanPrefix } from "shared-queue-keys"; @@ -61,6 +71,7 @@ const DELETE_COLLECT_READ_CONCURRENCY = 16; * hostsLosingNsOwnership: string[], * namespaceStillActive: boolean, * hasWorkerSecrets: boolean, + * hasWorkflowDefs: boolean, * }} DeleteInputs * @typedef {{ retained: boolean, objects: number, doStorageId?: string }} DoStorageRetention * @typedef {{ taskId: string, prefixes: string[], source: Record, nowMs?: number }} CleanupIntent @@ -80,8 +91,37 @@ const DELETE_COLLECT_READ_CONCURRENCY = 16; class WholeDeleteError extends ControlAbort {} -// Raised when an under-WATCH drift check observes state that disagrees -// with the pre-session snapshot — caller loops to re-collect. +/** @param {unknown} raw @param {string} host @param {string} slot */ +function requirePatternProjection(raw, host, slot) { + const projection = decodePatternProjection(raw); + if (!projection) { + throw new WholeDeleteError(500, "corrupt_pattern_projection", { + host, + slot, + stage: "pattern_projection_parse", + }); + } + return projection; +} + +/** @param {string} ns @param {string} name @returns {never} */ +function throwWholeDeleteContention(ns, name) { + throw new WholeDeleteError(503, "whole_delete_contention", { + namespace: ns, name, + message: `exhausted ${MAX_DELETE_ATTEMPTS} attempts; retry later`, + }); +} + +/** @param {string} ns @param {string} name @returns {never} */ +function throwDeleteLockExpired(ns, name) { + throw new WholeDeleteError(409, "deleting", { + namespace: ns, name, + message: "worker delete lock expired; retry the request", + }); +} + +// Raised when collection or an under-WATCH check observes state that +// changed across reads; callers retry from a fresh snapshot. class DriftSignal extends Error { /** @param {string} reason */ constructor(reason) { super(reason); this.name = "DriftSignal"; } @@ -101,7 +141,7 @@ export async function handle({ request, env: _env, url, ns, name, principal, req // client's pipe closes cleanly. await request.text().catch(() => {}); - const lockToken = await acquireDeleteLock(redis, ns, name); + const lockToken = await acquireDeleteLock(redis, ns, name, WHOLE_DELETE_LOCK_KIND); if (!lockToken) { log("warn", "worker_delete_rejected", { request_id: requestId, @@ -117,16 +157,15 @@ export async function handle({ request, env: _env, url, ns, name, principal, req let outcome; try { outcome = await executeWholeDelete({ - redis, ns, name, principal, requestId, log, + redis, ns, name, principal, requestId, log, lockToken, }); } catch (err) { if (err instanceof ControlAbort) { - log("warn", "worker_delete_rejected", { + log(err.status >= 500 ? "error" : "warn", "worker_delete_rejected", { request_id: requestId, namespace: ns, worker: name, - status: err.status, - reason: err.code, + ...codedErrorLogFields(err), }); return controlAbortResponse(err); } @@ -204,19 +243,36 @@ async function handleDryRun({ redis, ns, name, principal, requestId, log }) { let collected = null; let workflowBlocker = null; try { - collected = await collectDeleteInputs({ redis, ns, name }); - await assertWorkflowDeleteAllowed({ ns, worker: name }); + collected = await withOptimisticRetries( + async () => { + const snapshot = await collectDeleteInputs({ redis, ns, name }); + await assertDryRunActiveVersionRetained(redis, snapshot); + return snapshot; + }, + { + attempts: MAX_DELETE_ATTEMPTS, + isRetryableError: (err) => err instanceof DriftSignal, + onExhausted: () => throwWholeDeleteContention(ns, name), + } + ); + await assertWorkflowDeleteAllowed({ ns, worker: name, requestId }); } catch (err) { if (err instanceof WholeDeleteError) { - log("warn", "worker_dry_run_rejected", { + log(err.status >= 500 ? "error" : "warn", "worker_dry_run_rejected", { request_id: requestId, namespace: ns, worker: name, - status: err.status, reason: err.code, + ...codedErrorLogFields(err), }); return controlAbortResponse(err, { dryRun: true }); } if (err instanceof ControlAbort && err.code === "workflow_instances_active") { workflowBlocker = err; } else if (err instanceof ControlAbort) { + log(err.status >= 500 ? "error" : "warn", "worker_dry_run_rejected", { + request_id: requestId, + namespace: ns, + worker: name, + ...codedErrorLogFields(err), + }); return controlAbortResponse(err, { dryRun: true }); } else { throw err; @@ -232,7 +288,8 @@ async function handleDryRun({ redis, ns, name, principal, requestId, log }) { const noop = !collected.activeVersion && collected.retainedVersions.length === 0 && - !collected.hasWorkerSecrets; + !collected.hasWorkerSecrets && + !collected.hasWorkflowDefs; log("info", "worker_delete_dry_run", { request_id: requestId, @@ -255,6 +312,7 @@ async function handleDryRun({ redis, ns, name, principal, requestId, log }) { queueConsumersRemoved: collected.queueConsumerKeys.length, namespaceStillActive: collected.namespaceStillActive, hasWorkerSecrets: collected.hasWorkerSecrets, + hasWorkflowDefs: collected.hasWorkflowDefs, durableObjects: { storageRetention: describeDoStorageRetention(collected), }, @@ -280,15 +338,16 @@ async function handleDryRun({ redis, ns, name, principal, requestId, log }) { } /** - * @param {{ redis: RedisClient, ns: string, name: string, principal?: AccessPrincipal | null, requestId: string, log: ControlLogger }} args + * @param {{ redis: RedisClient, ns: string, name: string, principal?: AccessPrincipal | null, requestId: string, log: ControlLogger, lockToken: string }} args * @returns {Promise} */ -async function executeWholeDelete({ redis, ns, name, principal, requestId, log }) { - for (let attempt = 0; attempt < MAX_DELETE_ATTEMPTS; attempt++) { +async function executeWholeDelete({ redis, ns, name, principal, requestId, log, lockToken }) { + return await withOptimisticRetries(async () => { const collected = await collectDeleteInputs({ redis, ns, name }); + assertActiveVersionRetained(collected); let workflowBlocker = null; try { - await assertWorkflowDeleteAllowed({ ns, worker: name, allowCleanup: true }); + await assertWorkflowDeleteAllowed({ ns, worker: name, allowCleanup: true, requestId }); } catch (err) { if (err instanceof ControlAbort && err.code === "workflow_instances_active") { workflowBlocker = err; @@ -300,12 +359,16 @@ async function executeWholeDelete({ redis, ns, name, principal, requestId, log } const hasWorkerLifecycle = collected.activeVersion || collected.retainedVersions.length > 0 || - collected.hasWorkerSecrets; + collected.hasWorkerSecrets || + collected.hasWorkflowDefs; if (workflowBlocker && !hasWorkerLifecycle) { throw workflowBlocker; } if (!hasWorkerLifecycle) { - await deleteResidualDoRedis(redis, collected); + if (!await renewDeleteLock(redis, ns, name, lockToken)) { + throwDeleteLockExpired(ns, name); + } + await deleteResidualDoRedis(redis, collected, lockToken); await cleanupDoAlarmsOrWarn({ ns, worker: name, doStorageId: collected.doStorageId, requestId, log, }); @@ -337,28 +400,52 @@ async function executeWholeDelete({ redis, ns, name, principal, requestId, log } blockers, }); } - - try { - const result = await runSessionEXEC({ - redis, ns, name, principal, requestId, collected, - }); - await cleanupDoAlarmsOrWarn({ - ns, worker: name, doStorageId: collected.doStorageId, requestId, log, - }); - result.doStorageRetention = describeDoStorageRetention(collected); - return result; - } catch (err) { - if (err instanceof DriftSignal) continue; - if (err instanceof WatchError) continue; - throw err; + if (!await renewDeleteLock(redis, ns, name, lockToken)) { + throwDeleteLockExpired(ns, name); } - } - throw new WholeDeleteError(503, "whole_delete_contention", { - namespace: ns, name, - message: `exhausted ${MAX_DELETE_ATTEMPTS} attempts; retry later`, + + const result = await runSessionEXEC({ + redis, ns, name, principal, requestId, collected, lockToken, + }); + await cleanupDoAlarmsOrWarn({ + ns, worker: name, doStorageId: collected.doStorageId, requestId, log, + }); + result.doStorageRetention = describeDoStorageRetention(collected); + return result; + }, { + attempts: MAX_DELETE_ATTEMPTS, + isRetryableError: (err) => err instanceof DriftSignal || err instanceof WatchError, + onExhausted: () => throwWholeDeleteContention(ns, name), + }); +} + +/** @param {DeleteInputs} collected */ +function assertActiveVersionRetained(collected) { + if (!collected.activeVersion || collected.retainedVersions.includes(collected.activeVersion)) return; + throw new WholeDeleteError(409, "projection_drift", { + namespace: collected.ns, + name: collected.name, + active_version: collected.activeVersion, + reason: "active_not_in_worker_versions", }); } +/** @param {RedisClient} redis @param {DeleteInputs} collected */ +async function assertDryRunActiveVersionRetained(redis, collected) { + if (!collected.activeVersion || collected.retainedVersions.includes(collected.activeVersion)) return; + const [retainedVersions, activeVersion] = await Promise.all([ + redis.zRange(workerVersionsKey(collected.ns, collected.name), 0, -1), + redis.hGet(routesKey(collected.ns), collected.name), + ]); + if ( + activeVersion !== collected.activeVersion || + !arraysShallowEqual(retainedVersions, collected.retainedVersions) + ) { + throw new DriftSignal("active version or retained versions changed during dry-run"); + } + assertActiveVersionRetained(collected); +} + /** @param {DeleteInputs} collected */ function describeDoStorageRetention(collected) { return { @@ -374,15 +461,14 @@ function describeDoStorageRetention(collected) { */ async function scanDoOwnerKeys(redis, doStorageId) { if (!doStorageId) return []; - /** @type {string[]} */ - const keys = []; + const keys = new Set(); let cursor = "0"; do { const [next, found] = await redis.scan(cursor, doOwnerScopeScanPatternForStorage(doStorageId), 100); - keys.push(...found); + for (const key of found) keys.add(key); cursor = next; } while (cursor !== "0"); - return keys; + return [...keys]; } /** @@ -391,36 +477,75 @@ async function scanDoOwnerKeys(redis, doStorageId) { * @param {string} name */ async function scanQueueConsumerKeysForWorker(redis, ns, name) { - /** @type {string[]} */ - const queueConsumerKeys = []; + const queueConsumerKeys = new Set(); const prefix = queueConsumerScanPrefix(ns); let cursor = "0"; do { - const [next, keys] = await redis.scan(cursor, `${prefix}*`, 100); + const [next, found] = await redis.scan(cursor, `${prefix}*`, 100); + const keys = [...new Set(found)]; const workers = typeof redis.hGetMany === "function" ? await redis.hGetMany(keys.map((k) => [k, "worker"])) : await mapConcurrent(keys, DELETE_COLLECT_READ_CONCURRENCY, (k) => redis.hGet(k, "worker") ); for (let i = 0; i < keys.length; i += 1) { - if (workers[i] === name) queueConsumerKeys.push(keys[i]); + if (workers[i] === name) queueConsumerKeys.add(keys[i]); } cursor = next; } while (cursor !== "0"); - return queueConsumerKeys; + return [...queueConsumerKeys]; } /** * @param {RedisClient} redis * @param {DeleteInputs} collected + * @param {string} lockToken */ -async function deleteResidualDoRedis(redis, collected) { - if (collected.doOwnerKeys.length) { - await redis.del(...collected.doOwnerKeys); - } - if (collected.doStorageId) { - await redis.del(doStorageIdKey(collected.ns, collected.name)); - } +async function deleteResidualDoRedis(redis, collected, lockToken) { + await redis.session(async (iso) => { + const lockKey = deleteLockKey(collected.ns, collected.name); + const storageKey = doStorageIdKey(collected.ns, collected.name); + await iso.watch( + lockKey, + routesKey(collected.ns), + workerVersionsKey(collected.ns, collected.name), + workerSecretsKey(collected.ns, collected.name), + workflowDefsKey(collected.ns, collected.name), + storageKey, + ...collected.doOwnerKeys, + ); + + if (await iso.get(lockKey) !== lockToken) { + await iso.unwatch(); + throwDeleteLockExpired(collected.ns, collected.name); + } + const activeVersion = await iso.hGet(routesKey(collected.ns), collected.name); + const retainedVersions = await iso.zRange(workerVersionsKey(collected.ns, collected.name), 0, -1); + const hasWorkerSecrets = (await iso.exists(workerSecretsKey(collected.ns, collected.name))) > 0; + const hasWorkflowDefs = (await iso.exists(workflowDefsKey(collected.ns, collected.name))) > 0; + if (activeVersion || retainedVersions.length > 0 || hasWorkerSecrets || hasWorkflowDefs) { + await iso.unwatch(); + throw new DriftSignal("worker lifecycle appeared during residual cleanup"); + } + if (await iso.get(storageKey) !== collected.doStorageId) { + await iso.unwatch(); + throw new DriftSignal("DO storage pointer changed during residual cleanup"); + } + const currentOwnerKeys = await scanDoOwnerKeys(iso, collected.doStorageId); + if (!arraysShallowEqual(currentOwnerKeys.toSorted(), collected.doOwnerKeys.toSorted())) { + await iso.unwatch(); + throw new DriftSignal("DO owner keys changed during residual cleanup"); + } + + if (!collected.doStorageId && collected.doOwnerKeys.length === 0) { + await iso.unwatch(); + return; + } + const multi = iso.multi(); + if (collected.doOwnerKeys.length) multi.del(...collected.doOwnerKeys); + if (collected.doStorageId) multi.del(storageKey); + await multi.exec(); + }); } /** @@ -429,7 +554,7 @@ async function deleteResidualDoRedis(redis, collected) { async function cleanupDoAlarmsOrWarn({ ns, worker, doStorageId, requestId, log }) { if (!doStorageId) return; try { - await cleanupDoAlarmsForWorker({ ns, worker, doStorageId }); + await cleanupDoAlarmsForWorker({ ns, worker, doStorageId, requestId }); } catch (err) { log("warn", "worker_do_alarm_cleanup_failed", { request_id: requestId, @@ -455,8 +580,8 @@ async function collectDeleteInputs({ redis, ns, name }) { const queueConsumerKeys = await scanQueueConsumerKeysForWorker(redis, ns, name); - // Corrupt meta is fail-closed: swallowing would leave S3 orphans and - // stranded reverse referrers after a half-delete. + // Metadata behind retained/active indexes is required: proceeding without + // it would omit S3 and reverse-ref cleanup from a successful delete. const retainedVersions = await redis.zRange(workerVersionsKey(ns, name), 0, -1); /** @type {Record} */ const prefixByVersion = {}; @@ -474,23 +599,26 @@ async function collectDeleteInputs({ redis, ns, name }) { return { ver, rawMeta, referrers }; }); for (const { ver, rawMeta, referrers } of retainedReads) { - if (rawMeta) { - let meta; - try { - meta = JSON.parse(rawMeta); - } catch (err) { - throw new WholeDeleteError(500, "corrupt_meta", { - namespace: ns, name, version: ver, - stage: "retained_meta_parse", - detail: errMessage(err), - }); - } - if (meta && meta.assets && typeof meta.assets.prefix === "string") { - prefixByVersion[ver] = meta.assets.prefix; + if (rawMeta == null) { + const currentVersions = await redis.zRange(workerVersionsKey(ns, name), 0, -1); + if (!currentVersions.includes(ver)) { + throw new DriftSignal("retained versions changed during collection"); } - d1RefsByVersion[ver] = extractD1Refs(meta && meta.bindings); - outgoingRefsByVersion[ver] = extractOutgoingRefs(meta && meta.bindings, ns); } + const meta = parseBundleMeta(rawMeta, { + ns, + worker: name, + version: ver, + makeError: ({ reason }) => new WholeDeleteError(500, "corrupt_meta", { + namespace: ns, name, version: ver, + stage: "retained_meta_parse", + detail: reason, + }), + }); + const assetPrefix = bundleAssetPrefix(meta); + if (assetPrefix !== null) prefixByVersion[ver] = assetPrefix; + d1RefsByVersion[ver] = extractD1Refs(meta.bindings); + outgoingRefsByVersion[ver] = extractOutgoingRefs(meta.bindings, ns); referrersByVersion[ver] = referrers; } @@ -501,46 +629,40 @@ async function collectDeleteInputs({ redis, ns, name }) { let activeRoutes = []; if (activeVersion) { const rawMeta = await redis.hGet(bundleKey(ns, name, activeVersion), "__meta__"); - if (rawMeta) { - let meta; - try { - meta = JSON.parse(rawMeta); - } catch (err) { - throw new WholeDeleteError(500, "corrupt_meta", { - namespace: ns, name, version: activeVersion, - stage: "active_meta_parse", - detail: errMessage(err), - }); + if (rawMeta == null) { + const currentActive = await redis.hGet(routesKey(ns), name); + if (currentActive !== activeVersion) { + throw new DriftSignal("active version changed during collection"); } - activeRoutes = normalizeActiveRoutes(meta, { - namespace: ns, - name, - version: activeVersion, - }); } + const meta = parseBundleMeta(rawMeta, { + ns, + worker: name, + version: activeVersion, + makeError: ({ reason }) => new WholeDeleteError(500, "corrupt_meta", { + namespace: ns, name, version: activeVersion, + stage: "active_meta_parse", + detail: reason, + }), + }); + activeRoutes = normalizeActiveRoutes(meta, { + namespace: ns, + name, + version: activeVersion, + }); } // A host stays in ns-hosts if any OTHER worker in this ns still holds a // pattern slot on it — SREMing because we just happen to own a slot // there would strand those other workers from their declared host. const hostsInActiveRoutes = [...new Set(activeRoutes.map((r) => r.host))]; - const hostsLosingNsOwnership = []; const hostPatternReads = await redis.hGetAllMany(hostsInActiveRoutes.map((host) => patternsKey(host))); - for (let i = 0; i < hostsInActiveRoutes.length; i += 1) { - const h = hostsInActiveRoutes[i]; - const patternsOnHost = hostPatternReads[i]; - const ourSlots = new Set( - activeRoutes.filter((r) => r.host === h).map((r) => r.slot) - ); - let someoneElseFromNs = false; - for (const [slot, raw] of Object.entries(patternsOnHost)) { - if (ourSlots.has(slot)) continue; - if (typeof raw !== "string") continue; - const parsed = decodePatternProjection(raw); - if (parsed && parsed.ns === ns) { someoneElseFromNs = true; break; } - } - if (!someoneElseFromNs) hostsLosingNsOwnership.push(h); - } + const hostsLosingNsOwnership = findHostsLosingNsOwnership( + ns, + activeRoutes, + hostsInActiveRoutes, + hostPatternReads, + ); // Drives the EXEC-time channel choice. Empty namespaces need routes:flush // because routes:invalidate would re-add the freshly SREM'd ns to @@ -549,6 +671,7 @@ async function collectDeleteInputs({ redis, ns, name }) { const namespaceStillActive = otherActive.length > 0; const hasWorkerSecrets = (await redis.exists(workerSecretsKey(ns, name))) > 0; + const hasWorkflowDefs = (await redis.exists(workflowDefsKey(ns, name))) > 0; return { ns, @@ -569,6 +692,7 @@ async function collectDeleteInputs({ redis, ns, name }) { hostsLosingNsOwnership, namespaceStillActive, hasWorkerSecrets, + hasWorkflowDefs, }; } @@ -640,6 +764,33 @@ function arraysShallowEqual(a, b) { return true; } +/** + * @param {string} ns + * @param {RouteSlot[]} activeRoutes + * @param {string[]} hosts + * @param {Array>} patternRecords + */ +function findHostsLosingNsOwnership(ns, activeRoutes, hosts, patternRecords) { + const losing = []; + for (let i = 0; i < hosts.length; i += 1) { + const host = hosts[i]; + const ourSlots = new Set( + activeRoutes.filter((route) => route.host === host).map((route) => route.slot) + ); + let siblingOwnsHost = false; + for (const [slot, raw] of Object.entries(patternRecords[i] || {})) { + if (ourSlots.has(slot)) continue; + const projection = requirePatternProjection(raw, host, slot); + if (projection.ns === ns) { + siblingOwnsHost = true; + break; + } + } + if (!siblingOwnsHost) losing.push(host); + } + return losing; +} + /** * @template T,U * @param {T[]} items @@ -657,18 +808,19 @@ async function mapConcurrent(items, concurrency, fn) { } /** - * @param {{ redis: RedisClient, ns: string, name: string, principal?: AccessPrincipal | null, requestId: string, collected: DeleteInputs }} args + * @param {{ redis: RedisClient, ns: string, name: string, principal?: AccessPrincipal | null, requestId: string, collected: DeleteInputs, lockToken: string }} args * @returns {Promise>} */ -async function runSessionEXEC({ redis, ns, name, principal, requestId, collected }) { +async function runSessionEXEC({ redis, ns, name, principal, requestId, collected, lockToken }) { return await redis.session(async (iso) => { const watchKeys = [ routesKey(ns), - `hosts:${ns}`, + hostsKey(ns), deleteLockKey(ns, name), doStorageIdKey(ns, name), workerVersionsKey(ns, name), workerSecretsKey(ns, name), + workflowDefsKey(ns, name), ...(collected.doObjectRegistry ? [collected.doObjectRegistry] : []), ...collected.doOwnerKeys, ...collected.affectedHosts.map((h) => patternsKey(h)), @@ -680,6 +832,11 @@ async function runSessionEXEC({ redis, ns, name, principal, requestId, collected ]; await iso.watch(...watchKeys); + if (await iso.get(deleteLockKey(ns, name)) !== lockToken) { + await iso.unwatch(); + throwDeleteLockExpired(ns, name); + } + // Drift guards — normally unreachable for sanctioned control writers once // the delete lock is held. Keep whole-worker delete fail-closed if // control-owned projections no longer match the collected snapshot. @@ -688,16 +845,40 @@ async function runSessionEXEC({ redis, ns, name, principal, requestId, collected await iso.unwatch(); throw new DriftSignal("worker_versions changed during delete"); } - const curActive = (await iso.hGet(routesKey(ns), name)) || null; + const curRoutes = await iso.hGetAll(routesKey(ns)); + const curActive = curRoutes[name] || null; if (curActive !== collected.activeVersion) { await iso.unwatch(); throw new DriftSignal("active version changed during delete"); } + if (curActive && !curVersions.includes(curActive)) { + await iso.unwatch(); + throw new WholeDeleteError(409, "projection_drift", { + namespace: ns, name, + active_version: curActive, + reason: "active_not_in_worker_versions", + }); + } + const currentNamespaceStillActive = Object.keys(curRoutes).some((worker) => worker !== name); + const currentPatternRecords = await iso.hGetAllMany( + collected.affectedHosts.map((host) => patternsKey(host)) + ); + const currentHostsLosingNsOwnership = findHostsLosingNsOwnership( + ns, + collected.activeRoutes, + collected.affectedHosts, + currentPatternRecords, + ); const curHasSecrets = (await iso.exists(workerSecretsKey(ns, name))) > 0; if (curHasSecrets !== collected.hasWorkerSecrets) { await iso.unwatch(); throw new DriftSignal("secrets presence changed during delete"); } + const curHasWorkflowDefs = (await iso.exists(workflowDefsKey(ns, name))) > 0; + if (curHasWorkflowDefs !== collected.hasWorkflowDefs) { + await iso.unwatch(); + throw new DriftSignal("workflow definitions presence changed during delete"); + } if (collected.doObjectRegistry) { const curDoObjectMembers = await iso.sMembers(collected.doObjectRegistry); if (!arraysShallowEqual(curDoObjectMembers.toSorted(), collected.doObjectMembers.toSorted())) { @@ -734,9 +915,9 @@ async function runSessionEXEC({ redis, ns, name, principal, requestId, collected for (const r of collected.activeRoutes) { const held = await iso.hGet(patternsKey(r.host), r.slot); - if (!held) continue; - const parsed = decodePatternProjection(held); - if (parsed && (parsed.ns !== ns || parsed.worker !== name)) { + if (held == null) continue; + const parsed = requirePatternProjection(held, r.host, r.slot); + if (parsed.ns !== ns || parsed.worker !== name) { await iso.unwatch(); throw new WholeDeleteError(409, "projection_drift", { host: r.host, slot: r.slot, owner: parsed, @@ -754,12 +935,17 @@ async function runSessionEXEC({ redis, ns, name, principal, requestId, collected } } + const commitInputs = { + ...collected, + hostsLosingNsOwnership: currentHostsLosingNsOwnership, + namespaceStillActive: currentNamespaceStillActive, + }; const { cleanupTaskId, cleanupIntent, dedupedPrefixes } = - buildWorkerDeleteCleanup(collected, requestId, buildS3CleanupTaskId); + buildWorkerDeleteCleanup(commitInputs, requestId, buildS3CleanupTaskId); const multi = iso.multi(); stageWorkerDelete(multi, { - collected, + collected: commitInputs, channels: { routes: ROUTES_CHANNEL, routesFlush: ROUTES_FLUSH_CHANNEL, @@ -775,7 +961,7 @@ async function runSessionEXEC({ redis, ns, name, principal, requestId, collected retainedVersions: collected.retainedVersions, affectedHosts: collected.affectedHosts, queueConsumersRemoved: collected.queueConsumerKeys.length, - namespaceStillActive: collected.namespaceStillActive, + namespaceStillActive: currentNamespaceStillActive, cleanupTaskId, cleanupIntent, dedupedPrefixes, diff --git a/control/handlers/deploy.js b/control/handlers/deploy.js index b8bee81..a9d103b 100644 --- a/control/handlers/deploy.js +++ b/control/handlers/deploy.js @@ -5,7 +5,8 @@ import { getControlS3, runOptimistic, stageBundleCommit, buildS3CleanupTaskId, recordS3CleanupIntent, - ControlAbort, controlAbortResponse, codedErrorResponse, + ControlAbort, controlAbortResponse, codedErrorLogFields, codedErrorResponse, + secretEnvelopeErrorResponse, } from "control-shared"; import { PLATFORM_TIER_RESERVED_NS } from "shared-auth-roles"; import { @@ -15,7 +16,9 @@ import { extractOutgoingRefs, deleteLockKey, doStorageIdKey, + parseBundleMeta, workflowDefsKey, + platformDomainFromEnv, } from "control-lib"; import { stageD1ReferrerAdds, @@ -37,9 +40,14 @@ import { parseCronList, parseQueueConsumers, } from "control-topology"; -import { formatVersion, parseVersion, bundleKey, routesKey } from "shared-version"; +import { formatVersion, parseVersion, bundleKey, nextVersionKey, routesKey } from "shared-version"; import { nsSecretsKey, workerSecretsKey } from "shared-secret-keys"; -import { isReservedNs, isValidRouteNs, ROUTES_ALLOWED_RESERVED_NS } from "shared-ns-pattern"; +import { + isReservedNs, + isValidRouteNs, + ROUTES_ALLOWED_RESERVED_NS, + WORKFLOW_KEY_RE, +} from "shared-ns-pattern"; import { putAsset, inferContentType } from "control-s3"; import { generateAssetsToken, assetsPrefixFor } from "shared-assets-token"; import { resolveDatabaseRefFrom } from "control-d1-store"; @@ -58,8 +66,18 @@ const MAX_COMMIT_ATTEMPTS = 5; const DEPLOY_JSON_BODY_MAX_BYTES = 32 * 1024 * 1024; const DEPLOY_ASSET_UPLOAD_CONCURRENCY = 8; +// Deploy keeps thin local wrappers even though they mirror ControlAbort-like +// fields: commit aborts need cleanup/log handling, while request errors are +// pre-commit shape rejections. class DeployAbort extends ControlAbort {} +/** @param {Record} details */ +function deployAbortLogContext(details) { + const { databaseId, ...context } = details; + if (databaseId !== undefined) context.database_id = databaseId; + return context; +} + /** * @typedef {import("control-shared").ControlLogger} ControlLogger * @typedef {import("shared-redis").RedisClient} RedisClient @@ -147,6 +165,16 @@ function isPlatformExportMeta(value) { ); } +/** @param {string} ns @param {string} worker @param {string} version @param {unknown} raw */ +function linkableBundleMeta(ns, worker, version, raw) { + return parseBundleMeta(raw, { + ns, + worker, + version, + makeError: ({ message }) => new LinkError(500, "corrupt_meta", message), + }); +} + /** @param {unknown} err @returns {DeployRequestError} */ function deployRequestErrorFromUnknown(err) { const record = err && typeof err === "object" ? /** @type {Record} */ (err) : {}; @@ -332,7 +360,9 @@ async function validateServiceBindingsPreflight({ redis, ns, name, bindings }) { if (cached) return await cached; const load = (async () => { const rawMeta = await redis.hGet(bundleKey(targetNs, worker, version), "__meta__"); - return rawMeta ? JSON.parse(rawMeta) : null; + return rawMeta == null + ? null + : linkableBundleMeta(targetNs, worker, version, rawMeta); })(); metaCache.set(key, load); return await load; @@ -364,15 +394,9 @@ async function collectPlatformExports(redis) { }))).flat(); const exportsByWorker = await Promise.all(routeEntries.map(async ({ ns, worker, version }) => { - /** @type {{ exports?: unknown }} */ - let meta; - try { - const rawMeta = await redis.hGet(bundleKey(ns, worker, version), "__meta__"); - if (!rawMeta) return []; - meta = JSON.parse(rawMeta); - } catch { - return []; - } + const rawMeta = await redis.hGet(bundleKey(ns, worker, version), "__meta__"); + if (rawMeta == null) return []; + const meta = linkableBundleMeta(ns, worker, version, rawMeta); if (!Array.isArray(meta.exports)) return []; const exportsList = /** @type {unknown[]} */ (meta.exports); return exportsList @@ -513,7 +537,7 @@ async function uploadDeployAssets({ ) ); if (firstFailure) { - const { assetPath, key, err } = firstFailure; + const { key, err } = firstFailure; log("error", "asset_upload_failed", { request_id: requestId, namespace: ns, @@ -531,7 +555,7 @@ async function uploadDeployAssets({ response: jsonError( 502, "asset_upload_failed", - `Asset upload failed for ${assetPath}: ${errMessage(err)}`, + "Asset upload failed", { ...(warnings.length ? { warnings } : {}) } ), }; @@ -704,20 +728,33 @@ async function commitPreparedDeploy({ } const warningDetails = warnings.length ? { warnings } : {}; if (err instanceof DeployAbort) { - log("warn", "deploy_rejected", { + log(err.status >= 500 ? "error" : "warn", "deploy_rejected", { request_id: requestId, namespace: ns, worker: name, version, - status: err.status, - reason: err.code, - ...err.details, + ...codedErrorLogFields(err, err.code, { context: deployAbortLogContext(err.details) }), }); return { response: controlAbortResponse(err, warningDetails) }; } if (err instanceof WorkerEnvBudgetError) return { response: codedErrorResponse(err, err.code, warningDetails) }; if (err instanceof WorkerCodeBudgetError) return { response: codedErrorResponse(err, err.code, warningDetails) }; - if (err instanceof SecretEnvelopeError) return { response: jsonError(503, err.code, err.message, warningDetails) }; + if (err instanceof SecretEnvelopeError) { + return { + response: secretEnvelopeErrorResponse({ + err, + log, + event: "deploy_rejected", + fields: { + request_id: requestId, + namespace: ns, + worker: name, + version, + }, + responseDetails: warningDetails, + }), + }; + } throw err; } return { commitDurationMs: Date.now() - commitStartedAt }; @@ -728,9 +765,7 @@ export async function handle({ request, env, ns, name, requestId }) { const redis = requireControlRedis(); const s3 = getControlS3(); const log = requireControlLog(); - const platformDomain = typeof env.PLATFORM_DOMAIN === "string" && env.PLATFORM_DOMAIN - ? env.PLATFORM_DOMAIN - : "workers.local"; + const platformDomain = platformDomainFromEnv(env); const parsed = await parseDeployRequestForHandler({ request, ns, platformDomain }); if (parsed.response) return parsed.response; @@ -744,7 +779,15 @@ export async function handle({ request, env, ns, name, requestId }) { deployRequest: parsed.deployRequest, }); } catch (err) { - if (err instanceof LinkError) return codedErrorResponse(err, "link_error"); + if (err instanceof LinkError) { + log(err.status >= 500 ? "error" : "warn", "deploy_rejected", { + request_id: requestId, + namespace: ns, + worker: name, + ...codedErrorLogFields(err, "link_error"), + }); + return codedErrorResponse(err, "link_error"); + } throw err; } const { bindings: mergedBindings, warnings } = platformResult; @@ -782,7 +825,7 @@ export async function handle({ request, env, ns, name, requestId }) { return deployAssetsS3NotConfiguredResponse(warningDetails); } - const num = await redis.incr(`worker:${ns}:${name}:next_version`); + const num = await redis.incr(nextVersionKey(ns, name)); const version = formatVersion(num); const uploadResult = await uploadDeployAssetsBeforeCommit({ @@ -866,7 +909,7 @@ async function scheduleDeployAbortCleanup({ warnings.push({ kind: "assets_cleanup_task_failed", prefix, - reason: errMessage(err), + reason: "cleanup_task_write_failed", }); } } @@ -1061,7 +1104,7 @@ async function validateOutgoingRefsForCommit(iso, { outgoingRefs }) { }); } const targetMeta = targetMetas[index]; - if (!targetMeta) { + if (typeof targetMeta !== "string" || targetMeta.length === 0) { throw new DeployAbort(409, "target_drift", { target: { ns: ref.targetNs, @@ -1088,26 +1131,16 @@ async function materializeCommittedMetadata(iso, { ns, name, prepared, resolvedD const workflowDefUpdates = []; if (Array.isArray(committedMeta.workflows) && committedMeta.workflows.length) { const defsKey = workflowDefsKey(ns, name); - const rawDefs = await iso.hMGet(defsKey, committedMeta.workflows.map((workflow) => workflow.name)); - for (const [index, workflow] of committedMeta.workflows.entries()) { - const rawDef = rawDefs[index]; + const existingDefRaws = await iso.hMGet( + defsKey, + committedMeta.workflows.map((workflow) => workflow.name) + ); + for (let index = 0; index < committedMeta.workflows.length; index += 1) { + const workflow = committedMeta.workflows[index]; + const existingDef = parsePersistedWorkflowDef(workflow.name, existingDefRaws[index]); let workflowKey; - if (rawDef) { - /** @type {JsonObject} */ - let parsed; - try { - parsed = /** @type {JsonObject} */ (JSON.parse(rawDef)); - } catch { - throw new DeployAbort(500, "workflow_definition_corrupt", { - workflow: workflow.name, - }); - } - if (typeof parsed?.workflowKey !== "string" || !/^wf_[0-9a-f]{32}$/.test(parsed.workflowKey)) { - throw new DeployAbort(500, "workflow_definition_corrupt", { - workflow: workflow.name, - }); - } - workflowKey = parsed.workflowKey; + if (existingDef) { + workflowKey = existingDef.workflowKey; } else { workflowKey = newWorkflowKey(); } @@ -1137,6 +1170,35 @@ async function materializeCommittedMetadata(iso, { ns, name, prepared, resolvedD return { committedMeta, workflowDefUpdates, doStorageId }; } +/** + * @param {string} workflowName + * @param {string | null | undefined} rawDef + * @returns {{ workflowKey: string } | null} + */ +function parsePersistedWorkflowDef(workflowName, rawDef) { + if (rawDef == null) return null; + /** @type {JsonObject | null} */ + let parsed; + try { + parsed = typeof rawDef === "string" + ? /** @type {JsonObject} */ (JSON.parse(rawDef)) + : null; + } catch { + parsed = null; + } + if ( + !parsed || + Array.isArray(parsed) || + typeof parsed.workflowKey !== "string" || + !WORKFLOW_KEY_RE.test(parsed.workflowKey) + ) { + throw new DeployAbort(500, "workflow_definition_corrupt", { + workflow: workflowName, + }); + } + return { workflowKey: parsed.workflowKey }; +} + /** * @param {DeployCommitMulti} multi * @param {{ diff --git a/control/handlers/hosts.js b/control/handlers/hosts.js index 5bdad0a..a96f6d5 100644 --- a/control/handlers/hosts.js +++ b/control/handlers/hosts.js @@ -2,11 +2,14 @@ import { jsonResponse, jsonError, readJsonBody, + codedErrorLogFields, codedErrorResponse, requireControlLog, requireControlRedis, } from "control-shared"; import { reconcileHosts, RoutingError } from "control-routing"; +import { platformDomainFromEnv } from "control-lib"; +import { hostsKey } from "shared-version"; /** * @param {{ request: Request, env: Record, method: string, nsName: string, requestId: string }} args @@ -16,7 +19,7 @@ export async function handle({ request, env, method, nsName, requestId }) { const log = requireControlLog(); if (method === "GET") { - const hosts = await redis.sMembers(`hosts:${nsName}`); + const hosts = await redis.sMembers(hostsKey(nsName)); return jsonResponse(200, { namespace: nsName, hosts: [...hosts].toSorted(), @@ -27,9 +30,7 @@ export async function handle({ request, env, method, nsName, requestId }) { if (parsed.response) return parsed.response; const body = /** @type {Record} */ (parsed.body); try { - const platformDomain = typeof env.PLATFORM_DOMAIN === "string" && env.PLATFORM_DOMAIN - ? env.PLATFORM_DOMAIN - : "workers.local"; + const platformDomain = platformDomainFromEnv(env); const hosts = await reconcileHosts(redis, nsName, body, platformDomain); log("info", "hosts_reconciled", { request_id: requestId, @@ -39,6 +40,11 @@ export async function handle({ request, env, method, nsName, requestId }) { return jsonResponse(200, { namespace: nsName, hosts }); } catch (err) { if (err instanceof RoutingError) { + log(err.status >= 500 ? "error" : "warn", "hosts_reconcile_rejected", { + request_id: requestId, + namespace: nsName, + ...codedErrorLogFields(err, "routing_error"), + }); return codedErrorResponse(err, "routing_error"); } throw err; diff --git a/control/handlers/ns-secrets.js b/control/handlers/ns-secrets.js index f7f1aac..9144088 100644 --- a/control/handlers/ns-secrets.js +++ b/control/handlers/ns-secrets.js @@ -1,13 +1,16 @@ import { jsonResponse, jsonError, + errMessage, requireControlLog, requireControlRedis, stringEnv, + codedErrorLogFields, codedErrorResponse, runOptimistic, ControlAbort, controlAbortResponse, + secretEnvelopeErrorResponse, } from "control-shared"; import { invalidSecretMutationKeyResponse, @@ -17,6 +20,7 @@ import { routesKey, workerVersionsKey } from "shared-version"; import { nsSecretsKey, workerSecretsKey } from "shared-secret-keys"; import { workersIndexKey } from "control-lib"; import { + BundleMetaError, WorkerEnvBudgetError, assertWorkerLoaderUserEnvBudget, assertWorkerVersionsUserEnvBudget, @@ -29,11 +33,45 @@ const MAX_NS_SECRET_ATTEMPTS = 5; class NamespaceSecretAbort extends ControlAbort {} -/** @param {unknown} err */ -function namespaceSecretMutationErrorResponse(err) { - if (err instanceof NamespaceSecretAbort) return controlAbortResponse(err); +/** + * @param {unknown} err + * @param {{ log: import("control-shared").ControlLogger, requestId: string, nsName: string, secretKey: string, method: string }} context + */ +function namespaceSecretMutationErrorResponse(err, { log, requestId, nsName, secretKey, method }) { + if (err instanceof NamespaceSecretAbort) { + log(err.status >= 500 ? "error" : "warn", "ns_secret_mutation_rejected", { + request_id: requestId, + namespace: nsName, + key: secretKey, + method, + ...codedErrorLogFields(err), + }); + return controlAbortResponse(err); + } + if (err instanceof BundleMetaError) { + log("error", "ns_secret_mutation_rejected", { + request_id: requestId, + namespace: nsName, + key: secretKey, + method, + ...codedErrorLogFields(err, err.code, { errorDetail: errMessage(err.cause) }), + }); + return codedErrorResponse(err, err.code); + } if (err instanceof WorkerEnvBudgetError) return codedErrorResponse(err, err.code); - if (err instanceof SecretEnvelopeError) return jsonError(503, err.code, err.message); + if (err instanceof SecretEnvelopeError) { + return secretEnvelopeErrorResponse({ + err, + log, + event: "ns_secret_mutation_rejected", + fields: { + request_id: requestId, + namespace: nsName, + key: secretKey, + method, + }, + }); + } return null; } @@ -182,12 +220,21 @@ export async function handle({ request, env, method, nsName, secretKey, requestI if (method === "PUT" && secretKey !== undefined) { const invalidKey = invalidSecretMutationKeyResponse(secretKey); if (invalidKey) return invalidKey; - const put = await readEncryptedSecretPutValue({ - request, - env, - hashKey: nsSecretHashKey, - fieldName: secretKey, - }); + let put; + try { + put = await readEncryptedSecretPutValue({ + request, + env, + hashKey: nsSecretHashKey, + fieldName: secretKey, + }); + } catch (err) { + const response = namespaceSecretMutationErrorResponse(err, { + log, requestId, nsName, secretKey, method, + }); + if (response) return response; + throw err; + } if ("response" in put) return put.response; try { await mutateNamespaceSecret({ @@ -200,7 +247,9 @@ export async function handle({ request, env, method, nsName, secretKey, requestI plaintext: put.plaintext, }); } catch (err) { - const response = namespaceSecretMutationErrorResponse(err); + const response = namespaceSecretMutationErrorResponse(err, { + log, requestId, nsName, secretKey, method, + }); if (response) return response; throw err; } @@ -225,7 +274,9 @@ export async function handle({ request, env, method, nsName, secretKey, requestI method: "DELETE", }); } catch (err) { - const response = namespaceSecretMutationErrorResponse(err); + const response = namespaceSecretMutationErrorResponse(err, { + log, requestId, nsName, secretKey, method, + }); if (response) return response; throw err; } diff --git a/control/handlers/promote.js b/control/handlers/promote.js index 4966246..050a04a 100644 --- a/control/handlers/promote.js +++ b/control/handlers/promote.js @@ -2,13 +2,14 @@ import { jsonResponse, jsonError, readJsonBody, - formatError, + codedErrorLogFields, codedErrorResponse, requireControlLog, requireControlRedis, } from "control-shared"; import { parseVersion } from "shared-version"; import { promoteWithRoutes, RoutingError } from "control-routing"; +import { platformDomainFromEnv } from "control-lib"; /** * @param {{ request: Request, env: Record, ns: string, name: string, requestId: string }} args @@ -27,9 +28,7 @@ export async function handle({ request, env, ns, name, requestId }) { } try { - const platformDomain = typeof env.PLATFORM_DOMAIN === "string" && env.PLATFORM_DOMAIN - ? env.PLATFORM_DOMAIN - : "workers.local"; + const platformDomain = platformDomainFromEnv(env); const result = await promoteWithRoutes(redis, ns, name, body.version, { log, requestId }); log("info", "worker_promoted", { request_id: requestId, @@ -48,13 +47,12 @@ export async function handle({ request, env, ns, name, requestId }) { }); } catch (err) { if (err instanceof RoutingError) { - log("warn", "worker_promote_rejected", { + log(err.status >= 500 ? "error" : "warn", "worker_promote_rejected", { request_id: requestId, namespace: ns, worker: name, version: body.version, - status: err.status, - ...formatError(err), + ...codedErrorLogFields(err, "routing_error"), ...(err.details.attempts ? { attempts: err.details.attempts } : {}), }); return codedErrorResponse(err, "routing_error"); diff --git a/control/handlers/secret-put.js b/control/handlers/secret-put.js index 548db58..62947a2 100644 --- a/control/handlers/secret-put.js +++ b/control/handlers/secret-put.js @@ -5,7 +5,7 @@ import { stringEnv, } from "control-shared"; import { validateSecretKey } from "control-lib"; -import { encryptSecretValue, SecretEnvelopeError } from "shared-secret-envelope"; +import { encryptSecretValue } from "shared-secret-envelope"; const SECRET_PUT_JSON_BODY_MAX_BYTES = 128 * 1024; @@ -44,15 +44,8 @@ export async function readEncryptedSecretPutValue({ request, env, hashKey, field if (Buffer.byteLength(body.value, "utf8") > 64 * 1024) { return { response: jsonError(400, "invalid_request", "secret value too large (max 64 KiB utf-8)") }; } - try { - return { - plaintext: body.value, - encrypted: await encryptSecretValue(body.value, { env: stringEnv(env), hashKey, fieldName }), - }; - } catch (err) { - if (err instanceof SecretEnvelopeError) { - return { response: jsonError(503, err.code, err.message) }; - } - throw err; - } + return { + plaintext: body.value, + encrypted: await encryptSecretValue(body.value, { env: stringEnv(env), hashKey, fieldName }), + }; } diff --git a/control/handlers/versions.js b/control/handlers/versions.js index b82e855..0c61ea2 100644 --- a/control/handlers/versions.js +++ b/control/handlers/versions.js @@ -1,10 +1,9 @@ import { jsonResponse, jsonError, - acquireDeleteLock, releaseDeleteLock, + acquireDeleteLock, releaseDeleteLock, renewDeleteLock, assertWorkflowDeleteAllowed, buildS3CleanupTaskId, recordCleanupIntentOrWarn, - ControlAbort, controlAbortResponse, - errMessage, + ControlAbort, codedErrorLogFields, controlAbortResponse, requireControlLog, requireControlRedis, runOptimistic, @@ -13,6 +12,9 @@ import { workerVersionsKey, referrersKey, extractD1Refs, extractOutgoingRefs, formatReferrerBlocker, + bundleAssetPrefix, + parseBundleMeta, + workflowDefsKey, } from "control-lib"; import { stageD1ReferrerRemovals, @@ -21,7 +23,13 @@ import { stageWorkerVersionIndexDelete, stageWorkerVersionIndexRemove, } from "control-lifecycle-indexes"; -import { parseVersion, bundleKey, routesKey } from "shared-version"; +import { + VERSION_DELETE_LOCK_KIND, + bundleKey, + deleteLockKey, + parseVersion, + routesKey, +} from "shared-version"; import { workerSecretsKey } from "shared-secret-keys"; const MAX_DELETE_ATTEMPTS = 5; @@ -68,7 +76,7 @@ async function handleDelete({ ns, name, version, principal, requestId }) { return jsonError(400, "invalid_version", `Version must be "v", got ${JSON.stringify(version)}`); } - const lockToken = await acquireDeleteLock(redis, ns, name); + const lockToken = await acquireDeleteLock(redis, ns, name, VERSION_DELETE_LOCK_KIND); if (!lockToken) { log("warn", "version_delete_rejected", { request_id: requestId, @@ -84,19 +92,26 @@ async function handleDelete({ ns, name, version, principal, requestId }) { try { let result; try { - await assertWorkflowDeleteAllowed({ ns, worker: name, version, allowCleanup: true }); + await assertWorkflowDeleteAllowed({ + ns, worker: name, version, allowCleanup: true, requestId, + }); + if (!await renewDeleteLock(redis, ns, name, lockToken)) { + throw new VersionDeleteError(409, "deleting", { + namespace: ns, name, version, + message: "worker delete lock expired; retry the request", + }); + } result = await executeVersionDelete({ - redis, ns, name, version, principal, requestId, + redis, ns, name, version, principal, requestId, lockToken, }); } catch (err) { if (err instanceof ControlAbort) { - log("warn", "version_delete_rejected", { + log(err.status >= 500 ? "error" : "warn", "version_delete_rejected", { request_id: requestId, namespace: ns, worker: name, version, - status: err.status, - reason: err.code, + ...codedErrorLogFields(err), }); return controlAbortResponse(err); } @@ -137,9 +152,9 @@ async function handleDelete({ ns, name, version, principal, requestId }) { } /** - * @param {{ redis: RedisClient, ns: string, name: string, version: string, principal?: AccessPrincipal | null, requestId: string }} args + * @param {{ redis: RedisClient, ns: string, name: string, version: string, principal?: AccessPrincipal | null, requestId: string, lockToken: string }} args */ -async function executeVersionDelete({ redis, ns, name, version, principal, requestId }) { +async function executeVersionDelete({ redis, ns, name, version, principal, requestId, lockToken }) { return await runOptimistic(redis, { attempts: MAX_DELETE_ATTEMPTS, onExhausted: () => { @@ -150,13 +165,23 @@ async function executeVersionDelete({ redis, ns, name, version, principal, reque }, }, async (iso) => { await iso.watch( + deleteLockKey(ns, name), routesKey(ns), workerVersionsKey(ns, name), workerSecretsKey(ns, name), + workflowDefsKey(ns, name), bundleKey(ns, name, version), referrersKey(ns, name, version), ); + if (await iso.get(deleteLockKey(ns, name)) !== lockToken) { + await iso.unwatch(); + throw new VersionDeleteError(409, "deleting", { + namespace: ns, name, version, + message: "worker delete lock expired; retry the request", + }); + } + const currentActive = await iso.hGet(routesKey(ns), name); if (currentActive === version) { throw new VersionDeleteError(409, "active_version", { @@ -179,19 +204,16 @@ async function executeVersionDelete({ redis, ns, name, version, principal, reque } const bundleMetaRaw = await iso.hGet(bundleKey(ns, name, version), "__meta__"); - if (!bundleMetaRaw) { - throw new VersionDeleteError(404, "version_not_found", { - namespace: ns, name, version, - }); - } - let bundleMeta; - try { - bundleMeta = JSON.parse(bundleMetaRaw); - } catch { - throw new VersionDeleteError(500, "corrupt_meta", { + const bundleMeta = parseBundleMeta(bundleMetaRaw, { + ns, + worker: name, + version, + makeError: ({ reason }) => new VersionDeleteError(500, "corrupt_meta", { namespace: ns, name, version, - }); - } + stage: "target_meta_parse", + detail: reason, + }), + }); const referrerMembers = await iso.sMembers(referrersKey(ns, name, version)); if (referrerMembers.length > 0) { @@ -203,8 +225,7 @@ async function executeVersionDelete({ redis, ns, name, version, principal, reque const outgoingRefs = extractOutgoingRefs(bundleMeta.bindings, ns); const d1Refs = extractD1Refs(bundleMeta.bindings); - const prefix = bundleMeta.assets && bundleMeta.assets.prefix - ? bundleMeta.assets.prefix : null; + const prefix = bundleAssetPrefix(bundleMeta); // Secret-bump COPY can make siblings share the same assets prefix; // skip the cleanup task if any other retained version still points @@ -215,18 +236,17 @@ async function executeVersionDelete({ redis, ns, name, version, principal, reque for (const otherV of currentVersions) { if (otherV === version) continue; const otherRaw = await iso.hGet(bundleKey(ns, name, otherV), "__meta__"); - if (!otherRaw) continue; - let otherMeta; - try { - otherMeta = JSON.parse(otherRaw); - } catch (err) { - throw new VersionDeleteError(500, "corrupt_meta", { + const otherMeta = parseBundleMeta(otherRaw, { + ns, + worker: name, + version: otherV, + makeError: ({ reason }) => new VersionDeleteError(500, "corrupt_meta", { namespace: ns, name, version: otherV, stage: "sibling_meta_parse", - detail: errMessage(err), - }); - } - if (otherMeta && otherMeta.assets && otherMeta.assets.prefix === prefix) { + detail: reason, + }), + }); + if (bundleAssetPrefix(otherMeta) === prefix) { skipS3Cleanup = true; break; } @@ -234,6 +254,7 @@ async function executeVersionDelete({ redis, ns, name, version, principal, reque } const hasWorkerSecrets = (await iso.exists(workerSecretsKey(ns, name))) > 0; + const hasWorkflowDefs = (await iso.exists(workflowDefsKey(ns, name))) > 0; const lastRetainedNoActive = currentVersions.length === 1 && currentVersions[0] === version && @@ -253,11 +274,10 @@ async function executeVersionDelete({ redis, ns, name, version, principal, reque databaseIdFor: (ref) => /** @type {string} */ (ref.databaseId), }); if (lastRetainedNoActive) { - // Single-version DELETE must never touch worker secrets — that's - // whole-delete's job. A worker with secrets but no bundles / - // active projection stays in workers: as a secret-only entry. + // Single-version DELETE leaves non-version lifecycle state for + // whole-delete. Keep those workers discoverable through workers:. stageWorkerVersionIndexDelete(multi, ns, name); - if (!hasWorkerSecrets) { + if (!hasWorkerSecrets && !hasWorkflowDefs) { stageWorkerHidden(multi, ns, name); } } diff --git a/control/handlers/worker-secrets.js b/control/handlers/worker-secrets.js index 4824f77..a9434ed 100644 --- a/control/handlers/worker-secrets.js +++ b/control/handlers/worker-secrets.js @@ -3,15 +3,19 @@ import { jsonError, ControlAbort, controlAbortResponse, + errMessage, requireControlLog, requireControlRedis, runOptimistic, stringEnv, + codedErrorLogFields, codedErrorResponse, + secretEnvelopeErrorResponse, } from "control-shared"; import { deleteLockKey, routesKey, + workflowDefsKey, workerVersionsKey, } from "control-lib"; import { @@ -21,6 +25,7 @@ import { import { stageWorkerHidden, stageWorkerVisible } from "control-lifecycle-indexes"; import { bumpActiveAndPromote, RoutingError } from "control-routing"; import { + BundleMetaError, WorkerEnvBudgetError, assertWorkerVersionsUserEnvBudget, decryptMutatedSecretHashForBudget, @@ -48,6 +53,61 @@ const MAX_SECRET_ATTEMPTS = 5; class SecretAbort extends ControlAbort {} class SecretNoop extends Error {} +function secretMutationContentionAbort() { + return new SecretAbort(503, "secret_mutation_contention", { + message: "active version changed during secret mutation; retry later", + }); +} + +/** + * @param {{ + * abort: ControlAbort, + * requestId: string, + * ns: string, + * name: string, + * key: string, + * method: string, + * log: (level: string, event: string, fields: Record) => void, + * }} args + */ +function rejectSecretMutation({ abort, requestId, ns, name, key, method, log }) { + log(abort.status >= 500 ? "error" : "warn", "secret_mutation_rejected", { + request_id: requestId, + namespace: ns, + worker: name, + key, + method, + ...codedErrorLogFields(abort), + }); + return controlAbortResponse(abort); +} + +/** + * @param {{ + * err: SecretEnvelopeError, + * requestId: string, + * ns: string, + * name: string, + * key: string, + * method: string, + * log: (level: string, event: string, fields: Record) => void, + * }} args + */ +function rejectSecretEnvelopeMutation({ err, requestId, ns, name, key, method, log }) { + return secretEnvelopeErrorResponse({ + err, + log, + event: "secret_mutation_rejected", + fields: { + request_id: requestId, + namespace: ns, + worker: name, + key, + method, + }, + }); +} + /** * @param {{ * request: Request, @@ -78,12 +138,20 @@ export async function handle({ request, env, method, ns, name, subPath, requestI let storedValue = null; let putPlaintext = null; if (method === "PUT") { - const put = await readEncryptedSecretPutValue({ - request, - env, - hashKey: secretsKey, - fieldName: key, - }); + let put; + try { + put = await readEncryptedSecretPutValue({ + request, + env, + hashKey: secretsKey, + fieldName: key, + }); + } catch (err) { + if (err instanceof SecretEnvelopeError) { + return rejectSecretEnvelopeMutation({ err, requestId, ns, name, key, method, log }); + } + throw err; + } if ("response" in put) return put.response; storedValue = put.encrypted; putPlaintext = put.plaintext; @@ -173,57 +241,54 @@ export async function handle({ request, env, method, ns, name, subPath, requestI else payload.deleted = true; return jsonResponse(200, payload); } - throw new SecretAbort(503, "secret_mutation_contention", { - message: `active version changed during secret mutation; retry later`, - }); + throw secretMutationContentionAbort(); } catch (err) { if (err instanceof SecretAbort) { - log("warn", "secret_mutation_rejected", { + return rejectSecretMutation({ abort: err, requestId, ns, name, key, method, log }); + } + if (err instanceof BundleMetaError) { + log("error", "secret_mutation_rejected", { request_id: requestId, namespace: ns, worker: name, key, method, - status: err.status, - reason: err.code, + ...codedErrorLogFields(err, err.code, { errorDetail: errMessage(err.cause) }), }); - return controlAbortResponse(err); + return codedErrorResponse(err, err.code); } if (err instanceof WorkerEnvBudgetError) return codedErrorResponse(err, err.code); if (err instanceof RoutingError) { if (err.code === "bump_contention") { - const abort = new SecretAbort(503, "secret_mutation_contention", { - message: `active version changed during secret mutation; retry later`, - }); - log("warn", "secret_mutation_rejected", { - request_id: requestId, - namespace: ns, - worker: name, + return rejectSecretMutation({ + abort: secretMutationContentionAbort(), + requestId, + ns, + name, key, method, - status: abort.status, - reason: abort.code, + log, }); - return controlAbortResponse(abort); } if (err.code === "caller_deleting") { const abort = new SecretAbort(409, "deleting", { namespace: ns, worker: name, }); - log("warn", "secret_mutation_rejected", { - request_id: requestId, - namespace: ns, - worker: name, - key, - method, - status: abort.status, - reason: abort.code, - }); - return controlAbortResponse(abort); + return rejectSecretMutation({ abort, requestId, ns, name, key, method, log }); } + log(err.status >= 500 ? "error" : "warn", "secret_mutation_rejected", { + request_id: requestId, + namespace: ns, + worker: name, + key, + method, + ...codedErrorLogFields(err, "routing_error"), + }); return codedErrorResponse(err, err.code); } - if (err instanceof SecretEnvelopeError) return jsonError(503, err.code, err.message); + if (err instanceof SecretEnvelopeError) { + return rejectSecretEnvelopeMutation({ err, requestId, ns, name, key, method, log }); + } throw err; } } @@ -427,6 +492,7 @@ async function stageWorkerSecretForBump({ async function mutateSecretWithoutActive({ redis, ns, name, key, method, value, plaintext = null, controlEnv }) { const secretsKey = workerSecretsKey(ns, name); const nsSecretHashKey = nsSecretsKey(ns); + const defsKey = workflowDefsKey(ns, name); return await runOptimistic(redis, { attempts: MAX_SECRET_ATTEMPTS, onExhausted: () => { @@ -442,6 +508,7 @@ async function mutateSecretWithoutActive({ redis, ns, name, key, method, value, routesKey(ns), workerVersionsKey(ns, name), ]; + if (method === "DELETE") watches.push(defsKey); await iso.watch(...watches); const callerLock = await iso.get(deleteLockKey(ns, name)); @@ -463,7 +530,7 @@ async function mutateSecretWithoutActive({ redis, ns, name, key, method, value, } if (hkeys.length === 1 && hkeys[0] === key) { if (retainedVersions.length === 0) { - removeFromWorkersIndex = true; + removeFromWorkersIndex = (await iso.exists(defsKey)) === 0; } } } diff --git a/control/handlers/workers.js b/control/handlers/workers.js index 6cde730..8ed485b 100644 --- a/control/handlers/workers.js +++ b/control/handlers/workers.js @@ -1,5 +1,5 @@ import { jsonResponse, jsonError, requireControlLog, requireControlRedis } from "control-shared"; -import { routesKey, workersIndexKey, workerVersionsKey } from "control-lib"; +import { routesKey, workflowDefsKey, workersIndexKey, workerVersionsKey } from "control-lib"; import { workerSecretsKey } from "shared-secret-keys"; /** @param {{ method: string, nsName: string, requestId: string }} args */ @@ -18,8 +18,11 @@ export async function handle({ method, nsName, requestId }) { const versionKeys = names.map((name) => workerVersionsKey(nsName, name)); const secretKeys = names.map((name) => workerSecretsKey(nsName, name)); + const definitionKeys = names.map((name) => workflowDefsKey(nsName, name)); const versionsByWorker = await session.zRangeMany(versionKeys, 0, -1); - const secretFlags = await session.existsMany(secretKeys); + const stateFlags = await session.existsMany([...secretKeys, ...definitionKeys]); + const secretFlags = stateFlags.slice(0, names.length); + const definitionFlags = stateFlags.slice(names.length); return names.map((name, idx) => { const activeVersion = routesHash[name] || null; @@ -30,6 +33,7 @@ export async function handle({ method, nsName, requestId }) { versions, versionCount: versions.length, hasSecrets: Boolean(secretFlags[idx]), + hasWorkflowDefs: Boolean(definitionFlags[idx]), }; }); }); diff --git a/control/handlers/workflows.js b/control/handlers/workflows.js index fd190ee..dc5e034 100644 --- a/control/handlers/workflows.js +++ b/control/handlers/workflows.js @@ -3,28 +3,34 @@ import { WORKFLOW_NAME_RE, isValidWorkerName, isValidWorkflowName, + parseBundleMeta, workflowDefsKey, } from "control-lib"; import { - codedErrorResponse, - controlInternalJsonHeaders, + ControlAbort, + codedErrorLogFields, + controlAbortResponse, errMessage, - getControlWorkflows, jsonError, jsonResponse, + postWorkflowsInternal, requireControlLog, requireControlRedis, } from "control-shared"; import { bundleKey, routesKey } from "shared-version"; +import { + BINDING_NAME_RE, + WORKFLOW_KEY_RE, + isValidJsClassDeclarationName, +} from "shared-ns-pattern"; const LIFECYCLE_ACTIONS = new Set(["pause", "resume", "restart", "terminate"]); +const MAX_WORKFLOW_SNAPSHOT_ATTEMPTS = 2; /** * @typedef {import("shared-redis").RedisClient} RedisClient - * @typedef {{ fetch: typeof fetch }} WorkflowBackend * @typedef {{ method: string, url: URL, ns: string, subPath: string[], requestId: string }} WorkflowsHandlerArgs * @typedef {{ redis: RedisClient }} RedisDeps - * @typedef {{ redis: RedisClient, workflows: WorkflowBackend | null }} WorkflowDeps * @typedef {{ name: string, binding: string, className: string, workflowKey: string }} ActiveWorkflowMeta * @typedef {{ name: string, binding: string | null, className: string, workflowKey: string, retired?: boolean }} WorkflowEntry * @typedef {WorkflowEntry & { namespace: string, worker: string, activeVersion: string }} ListedWorkflowEntry @@ -50,8 +56,20 @@ export async function handle({ method, url, ns, subPath, requestId }) { try { return await handleInner({ method, url, ns, subPath, requestId }); } catch (err) { - if (err instanceof WorkflowControlError) { - return codedErrorResponse(err); + if (err instanceof ControlAbort) { + if (err.status >= 500) { + const detailWorker = typeof err.details?.worker === "string" + ? err.details.worker + : subPath[0]; + requireControlLog()("error", "workflow_request_rejected", { + request_id: requestId, + namespace: ns, + ...(detailWorker ? { worker: detailWorker } : {}), + ...(subPath[1] ? { workflow: subPath[1] } : {}), + ...codedErrorLogFields(err), + }); + } + return controlAbortResponse(err); } throw err; } @@ -60,9 +78,8 @@ export async function handle({ method, url, ns, subPath, requestId }) { /** @param {WorkflowsHandlerArgs} args */ async function handleInner({ method, url, ns, subPath, requestId }) { const redis = requireControlRedis(); - const workflows = getControlWorkflows(); const log = requireControlLog(); - const deps = { redis, workflows }; + const deps = { redis }; if (method === "GET" && subPath.length === 0) { const body = await listWorkflowDefinitions(deps, ns); log("info", "workflows_listed", { @@ -78,7 +95,7 @@ async function handleInner({ method, url, ns, subPath, requestId }) { const workflow = await resolveWorkflow(deps, ns, worker, workflowName); if (method === "GET" && subPath.length === 3) { - const body = await callWorkflowsRust(deps, "instances", { + const body = await callWorkflowsRust("instances", { ...workflow.request, options: listOptions(url), requestId, @@ -96,7 +113,7 @@ async function handleInner({ method, url, ns, subPath, requestId }) { if (subPath.length >= 4) { const instanceId = decodePathSegment(subPath[3], "workflow instance id"); if (method === "GET" && subPath.length === 4) { - const body = await callWorkflowsRust(deps, "status", { + const body = await callWorkflowsRust("status", { ...workflow.request, instanceId, options: statusOptions(url), @@ -116,9 +133,11 @@ async function handleInner({ method, url, ns, subPath, requestId }) { if (method === "POST" && subPath.length === 5 && LIFECYCLE_ACTIONS.has(subPath[4])) { const action = subPath[4]; if (workflow.workflow?.retired && action === "restart") { - throw new WorkflowControlError(409, "workflow_not_exported", `Workflow ${ns}/${worker}/${workflowName} is not exported by the active worker version`); + throw new ControlAbort(409, "workflow_not_exported", { + message: `Workflow ${ns}/${worker}/${workflowName} is not exported by the active worker version`, + }); } - const body = await callWorkflowsRust(deps, action, { + const body = await callWorkflowsRust(action, { ...workflow.request, instanceId, requestId, @@ -145,19 +164,50 @@ async function handleInner({ method, url, ns, subPath, requestId }) { * @param {string} ns */ async function listWorkflowDefinitions({ redis }, ns) { - const routes = await redis.hGetAll(routesKey(ns)); - /** @type {Array<[string, string]>} */ - const routeEntries = []; - for (const [worker, activeVersion] of Object.entries(routes)) { - if (typeof activeVersion === "string" && activeVersion) routeEntries.push([worker, activeVersion]); + for (let attempt = 0; attempt < MAX_WORKFLOW_SNAPSHOT_ATTEMPTS; attempt += 1) { + const routes = await redis.hGetAll(routesKey(ns)); + /** @type {Array<[string, string]>} */ + const routeEntries = []; + for (const [worker, activeVersion] of Object.entries(routes)) { + if (typeof activeVersion === "string" && activeVersion) routeEntries.push([worker, activeVersion]); + } + if (routeEntries.length === 0) return { namespace: ns, workflows: [] }; + const [metaRaws, defsRaws] = await readWorkflowListRaws({ redis }, ns, routeEntries); + const currentRoutes = await redis.hGetAll(routesKey(ns)); + /** @type {Array | undefined>} */ + const metas = new Array(routeEntries.length); + for (let index = 0; index < routeEntries.length; index += 1) { + const [worker, activeVersion] = routeEntries[index]; + if (currentRoutes[worker] === activeVersion) { + metas[index] = workflowBundleMeta(ns, worker, activeVersion, metaRaws[index]); + } + } + if (!sameRouteSnapshot(routes, currentRoutes)) continue; + return buildWorkflowDefinitionList( + ns, + routeEntries, + /** @type {Record[]} */ (metas), + defsRaws + ); } - if (routeEntries.length === 0) return { namespace: ns, workflows: [] }; - const [metaRaws, defsRaws] = await readWorkflowListRaws({ redis }, ns, routeEntries); + throw new ControlAbort(503, "workflow_metadata_contention", { + message: "Workflow metadata changed while it was being read", + namespace: ns, + }); +} + +/** + * @param {string} ns + * @param {Array<[string, string]>} routeEntries + * @param {Array>} metas + * @param {Array>} defsRaws + */ +function buildWorkflowDefinitionList(ns, routeEntries, metas, defsRaws) { /** @type {ListedWorkflowEntry[]} */ const workflows = []; for (let i = 0; i < routeEntries.length; i += 1) { const [worker, activeVersion] = routeEntries[i]; - const meta = parseBundleMetaRaw(ns, worker, activeVersion, metaRaws[i]); + const meta = metas[i]; const activeByName = new Map(); for (const workflow of workflowsFromMeta(meta)) { activeByName.set(workflow.name, workflow); @@ -171,7 +221,7 @@ async function listWorkflowDefinitions({ redis }, ns) { workflowKey: workflow.workflowKey, }); } - const defs = parseWorkflowDefs(defsRaws[i]); + const defs = parseWorkflowDefs(defsRaws[i], { ns, worker }); for (const [name, def] of Object.entries(defs)) { if (activeByName.has(name)) continue; workflows.push({ @@ -193,6 +243,12 @@ async function listWorkflowDefinitions({ redis }, ns) { return { namespace: ns, workflows: sortedWorkflows }; } +/** @param {Record} left @param {Record} right */ +function sameRouteSnapshot(left, right) { + const keys = Object.keys(left); + return keys.length === Object.keys(right).length && keys.every((key) => right[key] === left[key]); +} + /** * @param {RedisDeps} deps * @param {string} ns @@ -202,18 +258,40 @@ async function listWorkflowDefinitions({ redis }, ns) { */ async function resolveWorkflow({ redis }, ns, worker, workflowName) { if (!isValidWorkerName(worker)) { - throw new WorkflowControlError(400, "invalid_worker_name", `Invalid worker name ${JSON.stringify(worker)}. Must match ${WORKER_NAME_RE}.`); + throw new ControlAbort(400, "invalid_worker_name", { + message: `Invalid worker name ${JSON.stringify(worker)}. Must match ${WORKER_NAME_RE}.`, + }); } if (!isValidWorkflowName(workflowName)) { - throw new WorkflowControlError(400, "invalid_workflow_name", `Invalid workflow name ${JSON.stringify(workflowName)}. Must match ${WORKFLOW_NAME_RE}.`); - } - const activeVersion = await redis.hGet(routesKey(ns), worker); - if (!activeVersion) { - throw new WorkflowControlError(404, "worker_not_found", `Worker ${ns}/${worker} is not active`); + throw new ControlAbort(400, "invalid_workflow_name", { + message: `Invalid workflow name ${JSON.stringify(workflowName)}. Must match ${WORKFLOW_NAME_RE}.`, + }); } - const meta = await readBundleMeta({ redis }, ns, worker, activeVersion); - const workflow = workflowsFromMeta(meta).find((entry) => entry.name === workflowName); - if (workflow) { + for (let attempt = 0; attempt < MAX_WORKFLOW_SNAPSHOT_ATTEMPTS; attempt += 1) { + const active = await readActiveWorkflowMeta({ redis }, ns, worker); + if (!active) continue; + const { activeVersion, meta } = active; + const activeWorkflow = workflowsFromMeta(meta).find((entry) => entry.name === workflowName); + /** @type {WorkflowEntry | undefined} */ + let workflow = activeWorkflow; + if (!workflow) { + const def = await readWorkflowDef({ redis }, ns, worker, workflowName); + if (def) { + workflow = { + name: workflowName, + binding: null, + className: def.className, + workflowKey: def.workflowKey, + retired: true, + }; + } + } + if (await redis.hGet(routesKey(ns), worker) !== activeVersion) continue; + if (!workflow) { + throw new ControlAbort(404, "workflow_not_found", { + message: `Workflow ${ns}/${worker}/${workflowName} is not exported`, + }); + } return { workflow, request: { @@ -226,69 +304,90 @@ async function resolveWorkflow({ redis }, ns, worker, workflowName) { }, }; } + throw new ControlAbort(503, "workflow_metadata_contention", { + message: `Workflow metadata changed while ${ns}/${worker} was being read`, + namespace: ns, + worker, + }); +} - const defs = await readWorkflowDefs({ redis }, ns, worker); - const def = Object.hasOwn(defs, workflowName) - ? defs[workflowName] - : undefined; - if (!def) { - throw new WorkflowControlError(404, "workflow_not_found", `Workflow ${ns}/${worker}/${workflowName} is not exported`); +/** + * @param {RedisDeps} deps + * @param {string} ns + * @param {string} worker + * @returns {Promise<{ activeVersion: string, meta: Record } | null>} + */ +async function readActiveWorkflowMeta({ redis }, ns, worker) { + const activeVersion = await redis.hGet(routesKey(ns), worker); + if (!activeVersion) { + throw new ControlAbort(404, "worker_not_found", { + message: `Worker ${ns}/${worker} is not active`, + }); } - const retiredWorkflow = { - name: workflowName, - binding: null, - className: def.className, - workflowKey: def.workflowKey, - retired: true, - }; - return { - workflow: retiredWorkflow, - request: { - ns, - worker, - frozenVersion: activeVersion, - workflowName: retiredWorkflow.name, - workflowKey: retiredWorkflow.workflowKey, - className: retiredWorkflow.className, - }, - }; + const meta = await readBundleMeta({ redis }, ns, worker, activeVersion); + return meta ? { activeVersion, meta } : null; } /** * @param {RedisDeps} deps * @param {string} ns * @param {string} worker - * @returns {Promise} + * @param {string} workflowName + * @returns {Promise} + */ +async function readWorkflowDef({ redis }, ns, worker, workflowName) { + const raw = await redis.hGet(workflowDefsKey(ns, worker), workflowName); + return parseWorkflowDef(workflowName, raw, { ns, worker }); +} + +/** + * @param {string} name + * @param {string | null | undefined} value + * @param {{ ns: string, worker: string }} context + * @returns {RetiredWorkflowDef | null} */ -async function readWorkflowDefs({ redis }, ns, worker) { - return parseWorkflowDefs(await redis.hGetAll(workflowDefsKey(ns, worker))); +function parseWorkflowDef(name, value, context) { + if (value == null) return null; + let parsed; + try { + parsed = typeof value === "string" ? JSON.parse(value) : null; + } catch { + parsed = null; + } + if ( + !isValidWorkflowName(name) || + !parsed || + typeof parsed !== "object" || + Array.isArray(parsed) || + typeof parsed.workflowKey !== "string" || + !WORKFLOW_KEY_RE.test(parsed.workflowKey) || + !isValidJsClassDeclarationName(parsed.className) + ) { + throw new ControlAbort(500, "corrupt_meta", { + message: `Corrupt workflow definition for ${context.ns}/${context.worker}`, + namespace: context.ns, + worker: context.worker, + stage: "workflow_defs_parse", + }); + } + return { + workflowKey: parsed.workflowKey, + className: parsed.className, + }; } /** * @param {Record} raw + * @param {{ ns: string, worker: string }} context * @returns {RetiredWorkflowDefs} */ -function parseWorkflowDefs(raw) { +function parseWorkflowDefs(raw, context) { /** @type {RetiredWorkflowDefs} */ const defs = Object.create(null); for (const [name, value] of Object.entries(raw || {})) { - if (typeof value !== "string") continue; - try { - const parsed = JSON.parse(value); - if ( - parsed && - typeof parsed.workflowKey === "string" && - typeof parsed.className === "string" - ) { - defs[name] = { - workflowKey: parsed.workflowKey, - className: parsed.className, - }; - } - } catch { - // Corrupt retired definitions are ignored here; execution paths still - // fail closed inside workflows when they validate a concrete def. - } + const parsed = parseWorkflowDef(name, value, context); + if (!parsed) continue; + defs[name] = parsed; } return defs; } @@ -297,16 +396,57 @@ function parseWorkflowDefs(raw) { * @param {string} ns * @param {string} worker * @param {string} version - * @param {string | Uint8Array | null | undefined} raw - * @returns {unknown} + * @param {unknown} raw + * @returns {Record} */ -function parseBundleMetaRaw(ns, worker, version, raw) { - if (!raw) return null; - try { - return JSON.parse(String(raw)); - } catch { - throw new WorkflowControlError(500, "corrupt_meta", `Corrupt __meta__ for ${ns}/${worker}/${version}`); +function workflowBundleMeta(ns, worker, version, raw) { + const meta = parseBundleMeta(raw, { + ns, + worker, + version, + makeError: ({ message, reason }) => new ControlAbort(500, "corrupt_meta", { + message, + namespace: ns, + worker, + version, + stage: "bundle_meta_parse", + detail: reason, + }), + }); + const workflows = meta.workflows; + if (workflows !== undefined) { + if (!Array.isArray(workflows)) { + throw corruptWorkflowEntries(ns, worker, version); + } + const names = new Set(); + const bindings = new Set(); + const workflowKeys = new Set(); + for (const entry of workflows) { + if ( + !isActiveWorkflowMeta(entry) || + names.has(entry.name) || + bindings.has(entry.binding) || + workflowKeys.has(entry.workflowKey) + ) { + throw corruptWorkflowEntries(ns, worker, version); + } + names.add(entry.name); + bindings.add(entry.binding); + workflowKeys.add(entry.workflowKey); + } } + return meta; +} + +/** @param {string} ns @param {string} worker @param {string} version */ +function corruptWorkflowEntries(ns, worker, version) { + return new ControlAbort(500, "corrupt_meta", { + message: `Corrupt workflow metadata for ${ns}/${worker}/${version}`, + namespace: ns, + worker, + version, + stage: "workflow_entries_parse", + }); } /** @@ -329,7 +469,8 @@ async function readWorkflowListRaws({ redis }, ns, routeEntries) { worker_count: routeEntries.length, error_message: errMessage(err), }); - throw new WorkflowControlError(500, "workflow_metadata_unavailable", "Workflow metadata is unavailable", { + throw new ControlAbort(500, "workflow_metadata_unavailable", { + message: "Workflow metadata is unavailable", namespace: ns, worker_count: routeEntries.length, }); @@ -341,12 +482,13 @@ async function readWorkflowListRaws({ redis }, ns, routeEntries) { * @param {string} ns * @param {string} worker * @param {string} version - * @returns {Promise} + * @returns {Promise | null>} */ async function readBundleMeta({ redis }, ns, worker, version) { let raw; try { raw = await redis.hGet(bundleKey(ns, worker, version), "__meta__"); + if (raw == null && await redis.hGet(routesKey(ns), worker) !== version) return null; } catch (err) { requireControlLog()("error", "workflow_metadata_unavailable", { namespace: ns, @@ -354,13 +496,14 @@ async function readBundleMeta({ redis }, ns, worker, version) { version, error_message: errMessage(err), }); - throw new WorkflowControlError(500, "workflow_metadata_unavailable", "Workflow metadata is unavailable", { + throw new ControlAbort(500, "workflow_metadata_unavailable", { + message: "Workflow metadata is unavailable", namespace: ns, worker, version, }); } - return parseBundleMetaRaw(ns, worker, version, raw); + return workflowBundleMeta(ns, worker, version, raw); } /** @@ -372,7 +515,7 @@ function workflowsFromMeta(meta) { meta && typeof meta === "object" ? meta : null ); if (!record || !Array.isArray(record.workflows)) return []; - return record.workflows.filter(isActiveWorkflowMeta); + return /** @type {ActiveWorkflowMeta[]} */ (record.workflows); } /** @@ -385,40 +528,29 @@ function isActiveWorkflowMeta(entry) { ); return Boolean( record && - typeof record.name === "string" && - typeof record.binding === "string" && - typeof record.className === "string" && - typeof record.workflowKey === "string", + isValidWorkflowName(record.name) && + typeof record.binding === "string" && BINDING_NAME_RE.test(record.binding) && + isValidJsClassDeclarationName(record.className) && + typeof record.workflowKey === "string" && WORKFLOW_KEY_RE.test(record.workflowKey), ); } /** - * @param {WorkflowDeps} deps * @param {string} endpoint * @param {WorkflowRequest} body * @returns {Promise>} */ -async function callWorkflowsRust({ workflows }, endpoint, body) { - if (!workflows || typeof workflows.fetch !== "function") { - throw new WorkflowControlError(503, "workflow_internal_dispatch_failed", "Workflow backend is unavailable"); - } - let response; - let parsed; - try { - response = await workflows.fetch(`http://workflows/internal/workflows/${endpoint}`, { - method: "POST", - headers: controlInternalJsonHeaders(), - body: JSON.stringify(body), - }); - parsed = await response.json().catch(() => null); - } catch (err) { - requireControlLog()("error", "workflow_backend_request_failed", { - request_id: body.requestId || null, +async function callWorkflowsRust(endpoint, body) { + const { response, body: parsed } = await postWorkflowsInternal({ + endpoint: `workflows/${endpoint}`, + body, + requestId: body.requestId || null, + logEvent: "workflow_backend_request_failed", + logFields: { endpoint, - error_message: errMessage(err), - }); - throw new WorkflowControlError(503, "workflow_internal_dispatch_failed", "Workflow backend request failed"); - } + }, + timeoutMs: null, + }); if (!response.ok) { if (isUpstreamErrorBody(parsed)) { if (response.status >= 500) { @@ -432,17 +564,21 @@ async function callWorkflowsRust({ workflows }, endpoint, body) { } return throwUpstreamError(response.status, parsed); } - throw new WorkflowControlError( + throw new ControlAbort( response.status >= 400 ? response.status : 502, "workflow_internal_dispatch_failed", - "Workflow backend request failed", - { upstream_status: response.status }, + { + message: "Workflow backend request failed", + upstream_status: response.status, + }, ); } if (!parsed || typeof parsed !== "object") { - throw new WorkflowControlError(502, "workflow_internal_dispatch_failed", "Workflow backend returned an invalid response"); + throw new ControlAbort(502, "workflow_internal_dispatch_failed", { + message: "Workflow backend returned an invalid response", + }); } - return parsed; + return /** @type {Record} */ (parsed); } /** @@ -452,18 +588,22 @@ async function callWorkflowsRust({ workflows }, endpoint, body) { */ function throwUpstreamError(status, body) { if (status >= 500) { - throw new WorkflowControlError( + throw new ControlAbort( status, body.error, - "Workflow backend request failed", - { upstream_status: status }, + { + message: "Workflow backend request failed", + upstream_status: status, + }, ); } - throw new WorkflowControlError( + throw new ControlAbort( status, body.error, - typeof body.message === "string" ? body.message : body.error, - filterDetails(body), + { + ...filterDetails(body), + message: typeof body.message === "string" ? body.message : body.error, + }, ); } @@ -508,7 +648,9 @@ function statusOptions(url) { /** @type {Record} */ const options = {}; if (url.searchParams.has("include_steps") || url.searchParams.has("step_limit")) { - throw new WorkflowControlError(400, "invalid_request", "workflow status query options use camelCase"); + throw new ControlAbort(400, "invalid_request", { + message: "workflow status query options use camelCase", + }); } const includeSteps = url.searchParams.get("includeSteps"); if (includeSteps != null) options.includeSteps = parseBooleanOption(includeSteps, "includeSteps"); @@ -523,7 +665,7 @@ function statusOptions(url) { */ function parseIntegerOption(raw, label) { if (!/^(0|[1-9][0-9]*)$/.test(raw)) { - throw new WorkflowControlError(400, "invalid_request", `${label} must be an integer`); + throw new ControlAbort(400, "invalid_request", { message: `${label} must be an integer` }); } return Number(raw); } @@ -537,7 +679,7 @@ function parseBooleanOption(raw, label) { // numeric options stay strict because an empty number has no useful meaning. if (raw === "" || raw === "1" || raw === "true") return true; if (raw === "0" || raw === "false") return false; - throw new WorkflowControlError(400, "invalid_request", `${label} must be true or false`); + throw new ControlAbort(400, "invalid_request", { message: `${label} must be true or false` }); } /** @@ -548,21 +690,8 @@ function decodePathSegment(value, label) { try { return decodeURIComponent(value); } catch { - throw new WorkflowControlError(400, "invalid_request", `invalid percent-encoding in ${label}`); - } -} - -class WorkflowControlError extends Error { - /** - * @param {number} status - * @param {string} code - * @param {string} message - * @param {Record} [details] - */ - constructor(status, code, message, details = {}) { - super(message); - this.status = status; - this.code = code; - this.details = details; + throw new ControlAbort(400, "invalid_request", { + message: `invalid percent-encoding in ${label}`, + }); } } diff --git a/control/json-body.js b/control/json-body.js new file mode 100644 index 0000000..76c3795 --- /dev/null +++ b/control/json-body.js @@ -0,0 +1,47 @@ +import { BodyTooLargeError, readBoundedText } from "shared-bounded-body"; +import { jsonError } from "shared-respond"; + +export const DEFAULT_JSON_BODY_MAX_BYTES = 1024 * 1024; + +/** + * @param {Request} request + * @param {{ requireObject?: boolean, allowEmpty?: boolean, maxBytes?: number }} [opts] + */ +export async function readJsonBody( + request, + { requireObject = false, allowEmpty = false, maxBytes = DEFAULT_JSON_BODY_MAX_BYTES } = {}, +) { + let body; + try { + const text = await readBoundedText(request, maxBytes); + if (text === "") { + if (!allowEmpty) { + return { + response: jsonError(400, "invalid_json", "Body must be valid JSON"), + }; + } + body = {}; + } else { + body = JSON.parse(text); + } + } catch (err) { + if (err instanceof BodyTooLargeError) { + return { + response: jsonError( + 413, + "request_body_too_large", + `Body must be at most ${maxBytes} bytes` + ), + }; + } + return { + response: jsonError(400, "invalid_json", "Body must be valid JSON"), + }; + } + if (requireObject && (!body || typeof body !== "object" || Array.isArray(body))) { + return { + response: jsonError(400, "invalid_json_object", "Body must be a JSON object"), + }; + } + return { body }; +} diff --git a/control/lib.js b/control/lib.js index 71fa8d9..88e63db 100644 --- a/control/lib.js +++ b/control/lib.js @@ -1,9 +1,10 @@ -// Name grammar predicates, lifecycle Redis key helpers, referrer-index -// encode/decode, URL → {gate, ns} classifier, and secret-key validator. -// Pure data-shaping only. +// Name grammar predicates, lifecycle Redis key helpers, bundle metadata +// parsing, referrer-index encode/decode, URL → {gate, ns} classifier, +// and secret-key validation. Pure data-shaping only. import { NS_PATTERN, + RESERVED_OBJECT_KEYS, isReservedNs, RESERVED_TENANT_NS, WORKER_NAME_RE, @@ -11,23 +12,98 @@ import { QUEUE_NAME_RE, WDL_RESERVED_BINDING_RE, KV_ID_RE, + configuredHostname, + platformDomainFromEnv, validateModulePath, } from "shared-ns-pattern"; import { PLATFORM_TIER_RESERVED_NS, ROLES } from "shared-auth-roles"; -import { doStorageIdKey, routesKey, workerVersionsKey } from "shared-version"; -export { doStorageIdKey, routesKey, workerVersionsKey }; +import { errorMessage } from "shared-errors"; +import { + deleteLockKey, + doOwnerScopeScanPatternForStorage, + doStorageIdKey, + routesKey, + workerVersionsKey, +} from "shared-version"; +export { + deleteLockKey, + doOwnerScopeScanPatternForStorage, + doStorageIdKey, + routesKey, + workerVersionsKey, +}; export { validateModulePath }; export { WORKER_NAME_RE }; export { WORKFLOW_NAME_RE }; +export { configuredHostname, platformDomainFromEnv }; export const NS_RE = new RegExp(`^${NS_PATTERN}$`); export const MAX_QUEUE_DELAY_SECONDS = 86_400; +const WORKERD_DEPENDENCY_VERSION_RE = /^1\.(\d{4})(\d{2})(\d{2})\.(\d+)$/; /** * @typedef {{ callerNs: string, callerWorker: string, callerVersion: string, binding: string }} ReferrerMember * @typedef {{ kind?: string, ns?: string }} AccessPrincipal * @typedef {{ referrers: ReferrerMember[], crossNamespaceReferrerCount?: number }} ReferrerBlocker + * @typedef {{ namespace: string, worker: string, version: string, message: string, reason: string, cause: unknown }} BundleMetaFailure + */ + +export class BundleMetaError extends Error { + /** @param {{ namespace: string, worker: string, version: string, message: string, cause: unknown }} failure */ + constructor({ namespace, worker, version, message, cause }) { + super(message, { cause }); + this.name = "BundleMetaError"; + this.status = 500; + this.code = "corrupt_meta"; + this.details = { namespace, worker, version }; + } +} + +/** + * @param {unknown} raw + * @param {{ ns: string, worker: string, version: string, makeError?: (failure: BundleMetaFailure) => Error }} options + * @returns {Record} */ +export function parseBundleMeta(raw, { ns, worker, version, makeError }) { + /** @param {unknown} cause @returns {never} */ + const fail = (cause) => { + const failure = { + namespace: ns, + worker, + version, + message: `Corrupt __meta__ for ${ns}/${worker}/${version}`, + reason: errorMessage(cause), + cause, + }; + throw makeError ? makeError(failure) : new BundleMetaError(failure); + }; + + if (typeof raw !== "string") { + fail(new TypeError("__meta__ must be a JSON string")); + } + + let parsed; + try { + parsed = JSON.parse(raw); + } catch { + fail(new SyntaxError("__meta__ is not valid JSON")); + } + if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) { + fail(new TypeError("__meta__ must be a JSON object")); + } + return /** @type {Record} */ (parsed); +} + +/** + * @param {Record} meta + * @returns {string | null} + */ +export function bundleAssetPrefix(meta) { + const assets = meta.assets; + if (!assets || typeof assets !== "object" || Array.isArray(assets)) return null; + const prefix = /** @type {Record} */ (assets).prefix; + return typeof prefix === "string" ? prefix : null; +} /** @param {unknown} name */ export function isValidWorkerName(name) { @@ -76,14 +152,6 @@ export function projectAccessPrincipal(principal) { return null; } -/** @param {unknown} value */ -export function configuredHostname(value) { - if (typeof value !== "string" || !value.trim()) return null; - const host = value.trim().replace(/\.+$/, "").toLowerCase(); - if (!host || host.includes("/") || host.includes(":") || /\s/.test(host)) return null; - return host; -} - /** @param {unknown} value */ export function configuredPublicUrl(value) { if (typeof value !== "string" || !value.trim()) return null; @@ -100,18 +168,35 @@ export function configuredPublicUrl(value) { return parsed.href.replace(/\/+$/, ""); } -/** @param {string} source */ -export function platformVersionFromPackageJson(source) { +/** + * @param {string} source + * @returns {{ version: string, year: number, month: number, day: number, patch: number } | null} + */ +export function parseWorkerdDependencyVersion(source) { let parsed; try { parsed = JSON.parse(source); } catch { - return "wdl.unknown"; + return null; } const raw = parsed?.dependencies?.workerd; - if (typeof raw !== "string") return "wdl.unknown"; + if (typeof raw !== "string") return null; const version = raw.replace(/^[~^]/, ""); - return /^1\.\d{8}\.\d+$/.test(version) ? `wdl.${version.slice(2)}` : "wdl.unknown"; + const match = WORKERD_DEPENDENCY_VERSION_RE.exec(version); + if (!match) return null; + return { + version, + year: Number(match[1]), + month: Number(match[2]), + day: Number(match[3]), + patch: Number(match[4]), + }; +} + +/** @param {string} source */ +export function platformVersionFromPackageJson(source) { + const parsed = parseWorkerdDependencyVersion(source); + return parsed ? `wdl.${parsed.version.slice(2)}` : "wdl.unknown"; } // ─── Referrer index ──────────────────────────────────────────────── @@ -266,11 +351,6 @@ export function referrersKey(ns, worker, version) { return `worker-version-referrers:${ns}:${worker}:${version}`; } -/** @param {string} ns @param {string} worker */ -export function deleteLockKey(ns, worker) { - return `worker-delete-lock:${ns}:${worker}`; -} - /** @param {string} ns */ export function workersIndexKey(ns) { return `workers:${ns}`; @@ -311,11 +391,6 @@ export function d1DatabaseTombstonesKey(ns) { return `d1:database-tombstones:${ns}`; } -/** @param {string} storageId */ -export function doOwnerScopeScanPatternForStorage(storageId) { - return `do:owner:scope:${encodeURIComponent(`${storageId}:`)}*`; -} - /** @param {string} storageId */ export function doObjectRegistryKey(storageId) { return `do:objects:${encodeURIComponent(storageId)}`; @@ -623,5 +698,8 @@ export function validateSecretKey(key) { if (WDL_RESERVED_BINDING_RE.test(key)) { throw new Error("secret key is reserved for runtime-internal bindings"); } + if (RESERVED_OBJECT_KEYS.has(key)) { + throw new Error("secret key is a reserved Object.prototype key"); + } if (key.length > 128) throw new Error("secret key too long (max 128)"); } diff --git a/control/lifecycle-indexes.js b/control/lifecycle-indexes.js index 02f80f8..1610493 100644 --- a/control/lifecycle-indexes.js +++ b/control/lifecycle-indexes.js @@ -15,6 +15,8 @@ export const CRON_WORKER_INDEX_KEY = "cron:index:workers"; * @typedef {{ targetNs: string, targetWorker: string, targetVersion: string, binding: string }} OutgoingRef * @typedef {{ binding: string, databaseId?: string, resolvedDatabaseId?: string }} D1Ref * @typedef {{ id: string, gen: number, slot: number }} CronIndexEntry + * @typedef {{ cron: string, timezone: string }} CronSpec + * @typedef {{ cronSeq: number, addedWithPlacement: Array, removed: Array<{ id: string }> }} CronPlan * @typedef {{ queue: string, maxBatchSize: number, maxBatchTimeoutMs: number, maxRetries: number, retryDelaySeconds?: number, deadLetterQueue?: string }} QueueConsumer */ @@ -64,18 +66,33 @@ export function cronSlotKey(slotMs) { return `cron-slot:${slotMs}`; } +/** @param {number} slotMs */ +export function cronSlotExpireAt(slotMs) { + return Math.floor(slotMs / 1000) + 600; +} + /** @param {string} ns @param {string} worker @param {string} cronId @param {number} gen */ export function cronRefMember(ns, worker, cronId, gen) { return `${ns}:${worker}:${cronId}:${gen}`; } +/** @param {string} version @param {number} seq */ +export function cronMetaJson(version, seq) { + return JSON.stringify({ version, seq }); +} + +/** @param {CronSpec & { gen: number }} entry */ +export function cronEntryJson(entry) { + return JSON.stringify({ cron: entry.cron, timezone: entry.timezone, gen: entry.gen }); +} + /** @param {RedisMulti} multi @param {string} ns @param {string} worker @param {CronIndexEntry} entry */ export function stageCronSlotRef(multi, ns, worker, entry) { const key = cronSlotKey(entry.slot); multi.sAdd(key, cronRefMember(ns, worker, entry.id, entry.gen)); // Bound orphan refs (a crashed advance_ref leaves a member behind); // tick ignores slots >60s old, so 10min past slot wall-time is safe. - multi.expireAt(key, Math.floor(entry.slot / 1000) + 600); + multi.expireAt(key, cronSlotExpireAt(entry.slot)); } /** @param {RedisMulti} multi @param {string} ns @param {string} worker */ @@ -88,6 +105,35 @@ export function stageCronWorkerRemoved(multi, ns, worker) { multi.sRem(CRON_WORKER_INDEX_KEY, cronWorkerKey(ns, worker)); } +/** @param {RedisMulti} multi @param {{ ns: string, worker: string, version: string, cronKey: string, seq: number }} args */ +export function stageCronProjectionMeta(multi, { ns, worker, version, cronKey, seq }) { + stageCronWorkerIndexed(multi, ns, worker); + multi.hSet(cronKey, "__meta__", cronMetaJson(version, seq)); +} + +/** + * @param {RedisMulti} multi + * @param {{ ns: string, worker: string, version: string, cronKey: string, existingHash: Record, crons: CronSpec[], plan: CronPlan }} args + */ +export function stageCronProjection( + multi, + { ns, worker, version, cronKey, existingHash, crons, plan } +) { + // __meta__.version lets scheduler build x-worker-id without a second HGET; + // delete the hash when no crons remain so stale refs decay. + if (crons.length === 0) { + if (Object.keys(existingHash).length) multi.del(cronKey); + stageCronWorkerRemoved(multi, ns, worker); + return; + } + stageCronProjectionMeta(multi, { ns, worker, version, cronKey, seq: plan.cronSeq }); + for (const entry of plan.addedWithPlacement) { + multi.hSet(cronKey, entry.id, cronEntryJson(entry)); + stageCronSlotRef(multi, ns, worker, entry); + } + for (const removed of plan.removed) multi.hDel(cronKey, removed.id); +} + /** @param {RedisMulti} multi @param {{ ns: string, worker: string, version: string, refs: OutgoingRef[] }} args */ export function stageOutgoingReferrerAdds(multi, { ns, worker, version, refs }) { for (const ref of refs) { @@ -154,7 +200,7 @@ export function stageD1ReferrerRemovals(multi, { ns, worker, version, refs, data * @param {QueueConsumer} consumer * @returns {{ worker: string, version: string, max_batch_size: string, max_batch_timeout_ms: string, max_retries: string, dead_letter_queue?: string, retry_delay_secs?: string }} */ -function queueConsumerFields(worker, version, consumer) { +export function queueConsumerFields(worker, version, consumer) { /** @type {{ worker: string, version: string, max_batch_size: string, max_batch_timeout_ms: string, max_retries: string, dead_letter_queue?: string, retry_delay_secs?: string }} */ const fields = { worker, diff --git a/control/optimistic.js b/control/optimistic.js new file mode 100644 index 0000000..5573119 --- /dev/null +++ b/control/optimistic.js @@ -0,0 +1,28 @@ +import { WatchError } from "shared-redis"; +import { withOptimisticRetries } from "shared-optimistic-retry"; + +export { withOptimisticRetries }; + +/** + * @template T, S + * @param {{ session: (fn: (session: S) => Promise) => Promise }} redis + * @param {{ attempts?: number, onExhausted: () => unknown | Promise, onWatchError?: (err: unknown, attempt: number) => void, shouldRetryResult?: (result: T, attempt: number) => boolean }} options + * @param {(session: S, attempt: number) => Promise} fn + * @returns {Promise} + */ +export async function runOptimistic( + redis, + { attempts = 5, onExhausted, onWatchError, shouldRetryResult }, + fn +) { + return await withOptimisticRetries( + async (attempt) => await redis.session((session) => fn(session, attempt)), + { + attempts, + isRetryableError: (err) => err instanceof WatchError, + onRetryableError: onWatchError, + shouldRetryResult, + onExhausted: async () => /** @type {T} */ (await onExhausted()), + } + ); +} diff --git a/control/package.json b/control/package.json index 70731c9..6e46eb2 100644 --- a/control/package.json +++ b/control/package.json @@ -5,7 +5,7 @@ "author": "Sean Consulting OÜ", "type": "module", "dependencies": { - "@wdl-dev/aws-sigv4": "^2.0.0", + "@wdl-dev/aws-sigv4": "^3.0.1", "croner": "^10.0.1" } } diff --git a/control/r2.js b/control/r2.js index 40bf00a..454dc8a 100644 --- a/control/r2.js +++ b/control/r2.js @@ -1,5 +1,6 @@ import { SigV4Client } from "@wdl-dev/aws-sigv4"; import { discardResponseBody } from "shared-respond"; +import { S3_TRANSIENT_RETRIES } from "shared-s3-retry"; import { encodeS3KeyPath, encodeS3Query, @@ -13,7 +14,6 @@ import { import { collectXmlFields, listXmlTagValues, xmlTagValueIsTrue } from "shared-s3-xml"; const DEFAULT_LIST_LIMIT = 1000; -const S3_TRANSIENT_RETRIES = 10; /** * @typedef {{ client: SigV4Client, endpoint: string, bucket: string }} R2Admin @@ -60,6 +60,37 @@ function requestHeaders(requestId) { return headers; } +/** + * @param {{ r2: R2Admin, ns: string, bucketName: string, key: string, requestId?: string, method: "GET" | "HEAD" | "DELETE", notFound?: "error" | "null" | "ok" }} args + * @returns {Promise} + */ +async function fetchR2AdminObject({ + r2, + ns, + bucketName, + key, + requestId, + method, + notFound = "error", +}) { + const res = await r2.client.fetch(r2ObjectUrl(r2, { ns, bucketName }, key), { + method, + headers: requestHeaders(requestId), + }); + if (res.status === 404 && notFound !== "error") { + if (notFound === "null") { + await discardResponseBody(res); + return null; + } + return res; + } + if (!res.ok) { + const detail = await res.text().catch(() => ""); + throw new Error(`R2 admin ${method} failed with ${res.status}: ${detail.slice(0, 200)}`); + } + return res; +} + /** @param {unknown} value */ function limitFrom(value) { if (value == null || value === "") return DEFAULT_LIST_LIMIT; @@ -185,54 +216,42 @@ export async function listR2Objects({ /** @param {{ r2: R2Admin, ns: string, bucketName: string, key: string, requestId?: string }} args */ export async function getR2Object({ r2, ns, bucketName, key, requestId }) { - validateR2BucketName(bucketName); - normalizeR2ObjectKey(key); - const res = await r2.client.fetch(r2ObjectUrl(r2, { ns, bucketName }, key), { + return fetchR2AdminObject({ + r2, + ns, + bucketName, + key, + requestId, method: "GET", - headers: requestHeaders(requestId), + notFound: "null", }); - if (res.status === 404) { - await discardResponseBody(res); - return null; - } - if (!res.ok) { - const detail = await res.text().catch(() => ""); - throw new Error(`R2 admin GET failed with ${res.status}: ${detail.slice(0, 200)}`); - } - return res; } /** @param {{ r2: R2Admin, ns: string, bucketName: string, key: string, requestId?: string }} args */ export async function headR2Object({ r2, ns, bucketName, key, requestId }) { - validateR2BucketName(bucketName); - normalizeR2ObjectKey(key); - const res = await r2.client.fetch(r2ObjectUrl(r2, { ns, bucketName }, key), { + return fetchR2AdminObject({ + r2, + ns, + bucketName, + key, + requestId, method: "HEAD", - headers: requestHeaders(requestId), + notFound: "null", }); - if (res.status === 404) { - await discardResponseBody(res); - return null; - } - if (!res.ok) { - const detail = await res.text().catch(() => ""); - throw new Error(`R2 admin HEAD failed with ${res.status}: ${detail.slice(0, 200)}`); - } - return res; } /** @param {{ r2: R2Admin, ns: string, bucketName: string, key: string, requestId?: string }} args */ export async function deleteR2Object({ r2, ns, bucketName, key, requestId }) { - validateR2BucketName(bucketName); - normalizeR2ObjectKey(key); - const res = await r2.client.fetch(r2ObjectUrl(r2, { ns, bucketName }, key), { + const res = await fetchR2AdminObject({ + r2, + ns, + bucketName, + key, + requestId, method: "DELETE", - headers: requestHeaders(requestId), + notFound: "ok", }); - if (!res.ok && res.status !== 404) { - const detail = await res.text().catch(() => ""); - throw new Error(`R2 admin DELETE failed with ${res.status}: ${detail.slice(0, 200)}`); - } + if (!res) throw new Error("R2 admin DELETE unexpectedly returned no response"); await discardResponseBody(res); return { namespace: ns, bucket: bucketName, key, status: "ok" }; } diff --git a/control/routing.js b/control/routing.js index 89b144c..383af48 100644 --- a/control/routing.js +++ b/control/routing.js @@ -2,21 +2,17 @@ // stays connection-local; a nil EXEC raises WatchError and runOptimistic() // retries with fresh reads. -import { - DECLARED_HOSTS_KEY, - HOST_DECLARATIONS_PREFIX, - runOptimistic, -} from "control-shared"; +import { runOptimistic } from "control-shared"; import { d1DatabaseKey, deleteLockKey, extractD1Refs, extractOutgoingRefs, + parseBundleMeta, } from "control-lib"; import { cronWorkerKey, - stageCronSlotRef, - stageCronWorkerIndexed, - stageCronWorkerRemoved, + stageCronProjection, + stageCronProjectionMeta, stageD1ReferrerAdds, stageOutgoingReferrerAdds, stageQueueConsumerProjection, @@ -25,12 +21,27 @@ import { } from "control-lifecycle-indexes"; import { parseHostList } from "control-topology"; import { decodePatternProjection, encodePatternProjection } from "shared-route-projection"; -import { bundleKey, formatVersion, parseVersion, patternsKey, routesKey } from "shared-version"; +import { + DECLARED_HOSTS_KEY, + NAMESPACES_KEY, + PATTERNS_CHANNEL, + ROUTES_CHANNEL, + bundleKey, + formatVersion, + hostDeclarationsKey, + hostsKey, + nextVersionKey, + nsHostsKey, + parseVersion, + patternsKey, + routesKey, +} from "shared-version"; import { errorMessage } from "shared-errors"; import { diffCrons, nextFireMs, slotMsFor } from "control-cron-index"; import { isReservedNs, ROUTES_ALLOWED_RESERVED_NS } from "shared-ns-pattern"; import { PLATFORM_TIER_RESERVED_NS } from "shared-auth-roles"; import { queueConsumerKey } from "shared-queue-keys"; +import { WatchError } from "shared-redis"; import { computeAffectedHosts, computeNsHostDeltas, @@ -40,14 +51,6 @@ import { } from "control-routing-route-plan"; const MAX_ATTEMPTS = 5; -const PATTERNS_CHANNEL = "patterns:invalidate"; -const ROUTES_CHANNEL = "routes:invalidate"; - -/** @param {string} host */ -function hostDeclarationsKey(host) { - return `${HOST_DECLARATIONS_PREFIX}${host}`; -} - /** * @typedef {import("shared-route-projection").PatternProjection} PatternProjection * @typedef {Pick & { host: string, slot: string }} RoutePattern @@ -68,10 +71,12 @@ function hostDeclarationsKey(host) { * @typedef {{ routes?: RoutePattern[], crons?: CronSpec[], queueConsumers?: QueueConsumer[], exports?: ExportSpec[], bindings?: unknown }} BundleMeta * @typedef {Record>} HostState * @typedef {import("shared-redis").RedisMulti} RedisMulti - * @typedef {{ watch: (...keys: string[]) => Promise, unwatch: () => Promise, hGet: (key: string, field: string) => Promise, hGetMany: (pairs: Array<[string, string]>) => Promise>, hGetAll: (key: string) => Promise>, get: (key: string) => Promise, exists: (key: string) => Promise, sMIsMember: (key: string, ...members: string[]) => Promise, sMembers: (key: string) => Promise, zRange: (key: string, start: number, stop: number) => Promise, copy: (src: string, dst: string, options?: Record) => Promise, multi: () => RedisMulti }} RedisIso + * @typedef {{ watch: (...keys: string[]) => Promise, unwatch: () => Promise, hGet: (key: string, field: string) => Promise, hGetMany: (pairs: Array<[string, string]>) => Promise>, hGetAll: (key: string) => Promise>, hGetAllMany: (keys: string[]) => Promise>>, get: (key: string) => Promise, exists: (key: string) => Promise, sMIsMember: (key: string, ...members: string[]) => Promise, sMembers: (key: string) => Promise, zRange: (key: string, start: number, stop: number) => Promise, copy: (src: string, dst: string, options?: Record) => Promise, multi: () => RedisMulti }} RedisIso * @typedef {{ hGet: (key: string, field: string) => Promise, incr: (key: string) => Promise, session: (fn: (iso: RedisIso) => Promise) => Promise }} RedisClient */ +// Routing owns promotion/route-specific machine codes. Keep this separate from +// ControlAbort so pure routing helpers do not inherit handler abort semantics. export class RoutingError extends Error { /** @param {number} status @param {string} code @param {string} message @param {Record} [details] */ constructor(status, code, message, details = {}) { @@ -82,6 +87,20 @@ export class RoutingError extends Error { } } +/** @param {unknown} raw @param {string} host @param {string} slot */ +function requirePatternProjection(raw, host, slot) { + const projection = decodePatternProjection(raw); + if (!projection) { + throw new RoutingError( + 500, + "corrupt_pattern_projection", + `Pattern projection for ${host}${slot} is corrupt`, + { host, slot } + ); + } + return projection; +} + /** @param {Array} keys @returns {string[]} */ function uniqueKeys(keys) { const unique = new Set(); @@ -150,27 +169,32 @@ function parseCronMeta(raw, logContext) { async function readMeta(iso, ns, workerName, version) { if (!version) return null; const raw = await iso.hGet(bundleKey(ns, workerName, version), "__meta__"); - if (!raw) return null; - return parseBundleMeta(ns, workerName, version, raw); + if (raw == null) await retryIfRouteChanged(iso, ns, workerName, version); + return routingBundleMeta(ns, workerName, version, raw); +} + +/** @param {RedisIso} iso @param {string} ns @param {string} workerName @param {string} version */ +async function retryIfRouteChanged(iso, ns, workerName, version) { + // A watched route can move after its snapshot but before the dependent + // metadata read. Retry that race; a stable route still fails as corruption. + const currentVersion = await iso.hGet(routesKey(ns), workerName); + if (currentVersion !== version) throw new WatchError(); } /** * @param {string} ns * @param {string} workerName * @param {string} version - * @param {string} raw + * @param {unknown} raw */ -function parseBundleMeta(ns, workerName, version, raw) { - try { - return /** @type {BundleMeta} */ (JSON.parse(raw)); - } catch { - // Swallowing would hide old routes/consumers from the diff and leak them. - throw new RoutingError( - 500, - "corrupt_meta", - `Corrupt __meta__ for ${ns}/${workerName}/${version}` - ); - } +function routingBundleMeta(ns, workerName, version, raw) { + // Swallowing would hide old routes/consumers from the diff and leak them. + return /** @type {BundleMeta} */ (parseBundleMeta(raw, { + ns, + worker: workerName, + version, + makeError: ({ message }) => new RoutingError(500, "corrupt_meta", message), + })); } /** @param {RedisIso} iso @param {string} ns @param {string} workerName @param {string | null | undefined} version */ @@ -234,12 +258,12 @@ function computeCronPlan(cronHash, newCrons, now, logContext) { function stageDeclaredHostChanges(multi, ns, toAdd, removals) { const changedHosts = new Set([...toAdd, ...removals.map((entry) => entry.host)]); for (const host of toAdd) { - multi.sAdd(`hosts:${ns}`, host); + multi.sAdd(hostsKey(ns), host); multi.sAdd(hostDeclarationsKey(host), ns); multi.sAdd(DECLARED_HOSTS_KEY, host); } for (const { host, declaredNs } of removals) { - multi.sRem(`hosts:${ns}`, host); + multi.sRem(hostsKey(ns), host); multi.sRem(hostDeclarationsKey(host), ns); if (declaredNs.filter((declared) => declared !== ns).length === 0) { multi.del(hostDeclarationsKey(host)); @@ -249,30 +273,6 @@ function stageDeclaredHostChanges(multi, ns, toAdd, removals) { for (const host of changedHosts) multi.publish(PATTERNS_CHANNEL, host); } -/** @param {RedisMulti} multi @param {string} ns @param {string} workerName @param {string} newVersion @param {string} cronKey @param {Record} cronHash @param {CronSpec[]} newCrons @param {CronPlan} cronPlan */ -function stageCronPlan(multi, ns, workerName, newVersion, cronKey, cronHash, newCrons, cronPlan) { - // __meta__.version lets scheduler build x-worker-id without a - // second HGET; DEL when no crons remain so stale refs decay. - if (newCrons.length === 0) { - if (Object.keys(cronHash).length) multi.del(cronKey); - stageCronWorkerRemoved(multi, ns, workerName); - return; - } - stageCronWorkerIndexed(multi, ns, workerName); - multi.hSet(cronKey, "__meta__", JSON.stringify({ - version: newVersion, seq: cronPlan.cronSeq, - })); - for (const e of cronPlan.addedWithPlacement) { - multi.hSet(cronKey, e.id, JSON.stringify({ - cron: e.cron, timezone: e.timezone, gen: e.gen, - })); - stageCronSlotRef(multi, ns, workerName, e); - } - for (const r of cronPlan.removed) { - multi.hDel(cronKey, r.id); - } -} - /** @param {QueueConsumer[]} oldQueueConsumers @param {QueueConsumer[]} newQueueConsumers */ function computeQueueConsumerPlan(oldQueueConsumers, newQueueConsumers) { const newQueueSet = new Set(newQueueConsumers.map((c) => c.queue)); @@ -320,7 +320,7 @@ async function assertOutgoingDependenciesPresent(iso, targetLabel, outgoingRefs) bundleKey(ref.targetNs, ref.targetWorker, ref.targetVersion), "__meta__" ); - if (!targetMetaRaw) { + if (typeof targetMetaRaw !== "string" || targetMetaRaw.length === 0) { throw new RoutingError( 409, "service_binding_dependency_missing", @@ -367,7 +367,7 @@ async function assertQueueConsumerOwnership( async function assertDeclaredHosts(iso, ns, newRoutes) { const newHosts = [...new Set(newRoutes.map((r) => r.host))]; if (!newHosts.length) return; - const memberFlags = await iso.sMIsMember(`hosts:${ns}`, ...newHosts); + const memberFlags = await iso.sMIsMember(hostsKey(ns), ...newHosts); const missingIdx = memberFlags.findIndex((flag) => !flag); if (missingIdx !== -1) { const host = newHosts[missingIdx]; @@ -387,16 +387,20 @@ async function readHostStateAndAssertRouteConflicts(iso, ns, workerName, newRout const hostState = {}; /** @type {Map }>} */ const analysisByHost = new Map(); - for (const h of affectedHosts) { - const state = await iso.hGetAll(patternsKey(h)); + const hosts = [...affectedHosts]; + const states = hosts.length > 0 + ? await iso.hGetAllMany(hosts.map((host) => patternsKey(host))) + : []; + for (let index = 0; index < hosts.length; index += 1) { + const h = hosts[index]; + const state = states[index]; hostState[h] = state; /** @type {{ slot: string, held: PatternProjection } | null} */ let foreign = null; /** @type {Map} */ const slots = new Map(); for (const [slot, raw] of Object.entries(state)) { - const held = decodePatternProjection(raw); - if (!held) continue; + const held = requirePatternProjection(raw, h, slot); slots.set(slot, held); if (!foreign && held.ns && held.ns !== ns) foreign = { slot, held }; } @@ -452,10 +456,21 @@ async function assertPlatformAsAvailable(iso, ns, workerName, newExports) { for (let i = 0; i < routeEntries.length; i += 1) { const [otherWorker, otherVersion] = routeEntries[i]; const rawMeta = rawMetas[i]; - const otherMeta = typeof rawMeta === "string" - ? parseBundleMeta(otherNs, otherWorker, /** @type {string} */ (otherVersion), rawMeta) - : null; - if (!otherMeta || !Array.isArray(otherMeta.exports)) continue; + if (rawMeta == null) { + await retryIfRouteChanged( + iso, + otherNs, + otherWorker, + /** @type {string} */ (otherVersion) + ); + } + const otherMeta = routingBundleMeta( + otherNs, + otherWorker, + /** @type {string} */ (otherVersion), + rawMeta + ); + if (!Array.isArray(otherMeta.exports)) continue; const heldAs = new Set(); for (const e of otherMeta.exports) if (e.as) heldAs.add(e.as); for (const e of newExports) { @@ -486,10 +501,10 @@ async function readPromoteBundleInputs(iso, ns, workerName, newVersion) { // Every retry re-reads under WATCH: a concurrent single-version // delete (DEL bundle) surfaces here as a 404. const metaRaw = await iso.hGet(bundleKey(ns, workerName, newVersion), "__meta__"); - if (!metaRaw) { + if (metaRaw == null) { throw new RoutingError(404, "version_not_found", `Version ${newVersion} not found for ${ns}/${workerName}`); } - const meta = parseBundleMeta(ns, workerName, newVersion, metaRaw); + const meta = routingBundleMeta(ns, workerName, newVersion, metaRaw); const newRoutes = Array.isArray(meta.routes) ? meta.routes : []; const newCrons = Array.isArray(meta.crons) ? meta.crons : []; @@ -612,23 +627,22 @@ function stagePromoteWithRoutes(multi, ns, workerName, newVersion, inputs, obser stageVersionFlip(multi, ns, workerName, newVersion, inputs.newRoutes, observed.affectedHosts); // In the same MULTI as the route flip — closes the half-state where // routes: is set but the gateway's knownNs gate still 404s. - multi.sAdd("namespaces", ns); - if (plan.nsHostsAdd.length) multi.sAdd(`ns-hosts:${ns}`, plan.nsHostsAdd); - if (plan.nsHostsRem.length) multi.sRem(`ns-hosts:${ns}`, plan.nsHostsRem); + multi.sAdd(NAMESPACES_KEY, ns); + if (plan.nsHostsAdd.length) multi.sAdd(nsHostsKey(ns), plan.nsHostsAdd); + if (plan.nsHostsRem.length) multi.sRem(nsHostsKey(ns), plan.nsHostsRem); stageD1ReferrerAdds(multi, { ns, worker: workerName, version: newVersion, refs: inputs.d1Refs, databaseIdFor: (ref) => String(ref.databaseId), }); - stageCronPlan( - multi, + stageCronProjection(multi, { ns, - workerName, - newVersion, - plan.cronKey, - plan.cronHash, - inputs.newCrons, - plan.cronPlan - ); + worker: workerName, + version: newVersion, + cronKey: plan.cronKey, + existingHash: plan.cronHash, + crons: inputs.newCrons, + plan: plan.cronPlan, + }); stageQueueConsumerPlan(multi, ns, workerName, newVersion, plan.queuePlan); } @@ -653,7 +667,7 @@ export async function promoteWithRoutes(redis, ns, workerName, newVersion, optio }, async (iso) => { await iso.watch( routesKey(ns), - `hosts:${ns}`, + hostsKey(ns), cronKey, deleteLockKey(ns, workerName), bundleKey(ns, workerName, newVersion), @@ -695,7 +709,7 @@ export async function bumpActiveAndPromote(redis, ns, workerName, options = {}) ); } - const newNum = await redis.incr(`worker:${ns}:${workerName}:next_version`); + const newNum = await redis.incr(nextVersionKey(ns, workerName)); const newVersion = formatVersion(newNum); return await runOptimistic(redis, { @@ -711,7 +725,7 @@ export async function bumpActiveAndPromote(redis, ns, workerName, options = {}) }, async (iso) => { await iso.watch( routesKey(ns), - `hosts:${ns}`, + hostsKey(ns), deleteLockKey(ns, workerName), ); @@ -737,14 +751,16 @@ export async function bumpActiveAndPromote(redis, ns, workerName, options = {}) const dstKey = bundleKey(ns, workerName, newVersion); await iso.watch(srcKey, dstKey); - const srcMeta = await readMeta(iso, ns, workerName, currentVersion); - if (!srcMeta) { + const srcMetaRaw = await iso.hGet(srcKey, "__meta__"); + if (srcMetaRaw == null) { + await retryIfRouteChanged(iso, ns, workerName, currentVersion); throw new RoutingError( 500, "bundle_copy_failed", `COPY ${srcKey} → ${dstKey} failed (source missing?)` ); } + const srcMeta = routingBundleMeta(ns, workerName, currentVersion, srcMetaRaw); const routes = srcMeta && Array.isArray(srcMeta.routes) ? srcMeta.routes : []; const queueConsumers = srcMeta && Array.isArray(srcMeta.queueConsumers) ? srcMeta.queueConsumers : []; @@ -757,6 +773,13 @@ export async function bumpActiveAndPromote(redis, ns, workerName, options = {}) const affectedHosts = new Set(); for (const r of routes) affectedHosts.add(r.host); await watchKeys(iso, [...affectedHosts].map((h) => patternsKey(h))); + await readHostStateAndAssertRouteConflicts( + iso, + ns, + workerName, + routes, + affectedHosts + ); // Scheduler builds x-worker-id from cron __meta__.version. const cronKey = cronWorkerKey(ns, workerName); @@ -787,7 +810,7 @@ export async function bumpActiveAndPromote(redis, ns, workerName, options = {}) multi.copy(srcKey, dstKey, { REPLACE: true }); stageVersionFlip(multi, ns, workerName, newVersion, routes, affectedHosts); // Idempotent — also heals namespaces drift (manual SREM, recovery scripts). - multi.sAdd("namespaces", ns); + multi.sAdd(NAMESPACES_KEY, ns); // Maintain the indexes so a later hard-delete can collect this // bumped version's referrers. @@ -801,10 +824,13 @@ export async function bumpActiveAndPromote(redis, ns, workerName, options = {}) }); if (cronMetaParsed) { - stageCronWorkerIndexed(multi, ns, workerName); - multi.hSet(cronKey, "__meta__", JSON.stringify({ - version: newVersion, seq: cronSeqFromMeta(cronMetaParsed, logContext), - })); + stageCronProjectionMeta(multi, { + ns, + worker: workerName, + version: newVersion, + cronKey, + seq: cronSeqFromMeta(cronMetaParsed, logContext), + }); } for (const c of queueConsumers) { stageQueueConsumerProjection(multi, ns, workerName, newVersion, c); @@ -851,8 +877,8 @@ export async function reconcileHosts(redis, ns, body, platformDomain) { ); }, }, async (iso) => { - await iso.watch(`hosts:${ns}`); - const current = await iso.sMembers(`hosts:${ns}`); + await iso.watch(hostsKey(ns)); + const current = await iso.sMembers(hostsKey(ns)); const currentSet = new Set(current); const toAdd = [...bodySet.difference(currentSet)]; const toRemove = [...currentSet.difference(bodySet)]; @@ -867,15 +893,15 @@ export async function reconcileHosts(redis, ns, body, platformDomain) { // Reverse-index fast path; HGETALL only to enrich the 409 message. const reverseFlags = toRemove.length - ? await iso.sMIsMember(`ns-hosts:${ns}`, ...toRemove) + ? await iso.sMIsMember(nsHostsKey(ns), ...toRemove) : []; for (let i = 0; i < toRemove.length; i++) { const h = toRemove[i]; if (!reverseFlags[i]) continue; const entries = await iso.hGetAll(patternsKey(h)); for (const [slot, raw] of Object.entries(entries)) { - const parsed = decodePatternProjection(raw); - if (parsed && parsed.ns === ns) { + const parsed = requirePatternProjection(raw, h, slot); + if (parsed.ns === ns) { throw new RoutingError( 409, "host_in_use", diff --git a/control/s3.js b/control/s3.js index bed6fdb..1e8822f 100644 --- a/control/s3.js +++ b/control/s3.js @@ -3,6 +3,7 @@ import { SigV4Client } from "@wdl-dev/aws-sigv4"; import { encodeS3KeyPath } from "runtime-r2-utils"; +import { S3_TRANSIENT_RETRIES } from "shared-s3-retry"; const TYPE_BY_EXT = { ".html": "text/html; charset=utf-8", @@ -27,8 +28,6 @@ const TYPE_BY_EXT = { ".wasm": "application/wasm", ".map": "application/json", }; -const S3_TRANSIENT_RETRIES = 10; - /** * @typedef {{ client: SigV4Client, endpoint: string, bucket: string }} S3Client */ diff --git a/control/shared.js b/control/shared.js index 4e5e247..7b45265 100644 --- a/control/shared.js +++ b/control/shared.js @@ -1,18 +1,28 @@ // Shared state + helpers for control. Handlers import from here; the // dispatcher in index.js calls ensureInit(env) before route dispatch. -import { RedisClient, WatchError, redisDbFromEnv } from "shared-redis"; +import { RedisClient, redisDbFromEnv } from "shared-redis"; import { makeS3Client } from "control-s3"; import { makeR2AdminClient } from "control-r2"; import { extractToken } from "shared-auth-token"; import { validatePrincipalShape } from "shared-auth-roles"; -import { BodyTooLargeError, readBoundedText } from "shared-bounded-body"; import { queueStreamKey } from "shared-queue-keys"; import { internalErrorResponse, jsonError, jsonResponse, sanitizeJsonErrorDetails } from "shared-respond"; import { withInternalAuth } from "shared-internal-auth"; import { errorMessage } from "shared-errors"; import { randomHex } from "shared-random-id"; -import { deleteLockKey } from "control-lib"; +import { + DECLARED_HOSTS_KEY, + HOST_DECLARATIONS_SCAN_PATTERN, + HOSTS_SCAN_PATTERN, + PATTERNS_CHANNEL, + ROUTES_CHANNEL, + ROUTES_FLUSH_CHANNEL, + deleteLockKey, + formatDeleteLockToken, + hostDeclarationsKey, + namespaceFromHostsKey, +} from "shared-version"; import { S3_CLEANUP_QUEUE_NAME, S3_CLEANUP_TASK_ID_PREFIX, @@ -24,15 +34,31 @@ import { formatError, recordRedisCommand, } from "shared-observability"; +import { + WORKFLOWS_INTERNAL_TIMEOUT_MS, + createPostWorkflowsInternal, +} from "control-workflows-client"; +import { + ControlAbort, + controlAbortLogDetails, + codedErrorLogFields, + codedErrorResponse, + controlAbortResponse, + secretEnvelopeErrorResponse, +} from "control-errors"; +import { runOptimistic, withOptimisticRetries } from "control-optimistic"; +import { + DEFAULT_JSON_BODY_MAX_BYTES, + readJsonBody, +} from "control-json-body"; +import { + acquireTokenLock, + createTokenLock, + releaseTokenLock, + renewTokenLock, +} from "shared-redis-lock"; -export const ROUTES_CHANNEL = "routes:invalidate"; -export const ROUTES_FLUSH_CHANNEL = "routes:flush"; -export const PATTERNS_CHANNEL = "patterns:invalidate"; -export const DEFAULT_JSON_BODY_MAX_BYTES = 1024 * 1024; -export const DECLARED_HOSTS_KEY = "declared-hosts"; -export const HOST_DECLARATIONS_PREFIX = "host-declarations:"; -const HOSTS_PREFIX = "hosts:"; -const DO_ALARM_CLEANUP_TIMEOUT_MS = 5_000; +export { PATTERNS_CHANNEL, ROUTES_CHANNEL, ROUTES_FLUSH_CHANNEL }; /** * @typedef {import("shared-observability").RedisCommandEvent} RedisCommandEvent @@ -69,6 +95,16 @@ let s3Initialized = false; let r2Initialized = false; export { formatError, internalErrorResponse, jsonError, jsonResponse }; +export { + ControlAbort, + controlAbortLogDetails, + codedErrorLogFields, + codedErrorResponse, + controlAbortResponse, + secretEnvelopeErrorResponse, +}; +export { runOptimistic, withOptimisticRetries }; +export { DEFAULT_JSON_BODY_MAX_BYTES, readJsonBody }; /** @param {unknown} value @returns {value is Record} */ function isRecord(value) { @@ -101,30 +137,6 @@ export function prefixedId(prefix, bytes = 16) { return `${prefix}${randomHex(bytes)}`; } -/** - * @template T, S - * @param {{ session: (fn: (session: S) => Promise) => Promise }} redis - * @param {{ attempts?: number, onExhausted: () => unknown | Promise, onWatchError?: (err: unknown, attempt: number) => void, shouldRetryResult?: (result: T, attempt: number) => boolean }} options - * @param {(session: S, attempt: number) => Promise} fn - * @returns {Promise} - */ -export async function runOptimistic(redis, { attempts = 5, onExhausted, onWatchError, shouldRetryResult }, fn) { - for (let attempt = 0; attempt < attempts; attempt += 1) { - try { - const result = await redis.session((session) => fn(session, attempt)); - if (shouldRetryResult?.(result, attempt)) continue; - return result; - } catch (err) { - if (err instanceof WatchError) { - onWatchError?.(err, attempt); - continue; - } - throw err; - } - } - return /** @type {T} */ (await onExhausted()); -} - /** @param {Record} env */ export function ensureInit(env) { state.env = env; @@ -199,11 +211,6 @@ export function getControlR2() { return state.r2; } -/** @returns {WorkflowBackend | null} */ -export function getControlWorkflows() { - return state.workflows; -} - export function controlInternalJsonHeaders() { if (!state.env) { throw new Error("control shared state has not been initialized"); @@ -216,105 +223,11 @@ function onRedisCommand(event) { recordRedisCommand({ metrics: null, log: state.log, service: state.service, event }); } -export class ControlAbort extends Error { - /** - * @param {number} status - * @param {string} code - * @param {{ message?: string, [key: string]: unknown }} [details] - */ - constructor(status, code, details = {}) { - super(details?.message || code); - this.status = status; - this.code = code; - this.details = details; - } -} - -// Domain errors still flow through jsonError() so details cannot shadow the -// top-level wire contract. -/** - * @param {unknown} err - * @param {string} [fallbackCode] - * @param {Record} [extraDetails] - */ -export function codedErrorResponse(err, fallbackCode = "internal_error", extraDetails = {}) { - const record = isRecord(err) ? err : {}; - const details = isRecord(record.details) ? record.details : {}; - const status = typeof record.status === "number" ? record.status : 500; - const code = typeof record.code === "string" && record.code ? record.code : fallbackCode; - const recordMessage = typeof record.message === "string" && record.message ? record.message : undefined; - const message = err instanceof ControlAbort - ? (typeof err.details.message === "string" && err.details.message) || err.message || code - : recordMessage || (typeof details.message === "string" && details.message) || code; - return jsonError( - status, - code, - message, - { ...details, ...extraDetails }, - ); -} - -// ControlAbort always carries code; the separate wrapper keeps abort call sites -// explicit while ordinary coded errors still get a fallback code. -/** - * @param {ControlAbort} err - * @param {Record} [extraDetails] - */ -export function controlAbortResponse(err, extraDetails = {}) { - return codedErrorResponse(err, err.code, extraDetails); -} - -/** - * @param {Request} request - * @param {{ requireObject?: boolean, allowEmpty?: boolean, maxBytes?: number }} [opts] - */ -export async function readJsonBody( - request, - { requireObject = false, allowEmpty = false, maxBytes = DEFAULT_JSON_BODY_MAX_BYTES } = {}, -) { - let body; - try { - const limited = await readTextBody(request, maxBytes); - if ("response" in limited) return { response: limited.response }; - const text = limited.text; - if (text === "") { - if (!allowEmpty) { - return { - response: jsonError(400, "invalid_json", "Body must be valid JSON"), - }; - } - body = {}; - } else { - body = JSON.parse(text); - } - } catch { - return { - response: jsonError(400, "invalid_json", "Body must be valid JSON"), - }; - } - if (requireObject && (!body || typeof body !== "object" || Array.isArray(body))) { - return { - response: jsonError(400, "invalid_json_object", "Body must be a JSON object"), - }; - } - return { body }; -} - -/** - * @param {Request} request - * @param {number} maxBytes - * @returns {Promise<{ text: string } | { response: Response }>} - */ -async function readTextBody(request, maxBytes) { - try { - return { text: await readBoundedText(request, maxBytes) }; - } catch (err) { - if (err instanceof BodyTooLargeError) { - return { response: jsonError(413, "request_body_too_large", `Body must be at most ${maxBytes} bytes`) }; - } - throw err; - } -} +export const postWorkflowsInternal = createPostWorkflowsInternal({ + getWorkflows: () => state.workflows, + headers: controlInternalJsonHeaders, + getLog: () => state.log, +}); // AuthPolicyError → HTTP body shape. 4xx messages are user-actionable; // 5xx messages stay generic so internal auth diagnostics do not leak. @@ -424,11 +337,6 @@ async function scanKeys(redis, pattern) { return keys; } -/** @param {string} key */ -function namespaceFromHostsKey(key) { - return key.startsWith(HOSTS_PREFIX) ? key.slice(HOSTS_PREFIX.length) : ""; -} - /** * Rebuild the global declared-host gate from namespace-owned `hosts:` sets. * This is an ops reload repair path, not the ordinary writer; `/hosts` reconcile @@ -437,8 +345,8 @@ function namespaceFromHostsKey(key) { * @param {RedisClient} [redis] */ export async function rebuildDeclaredHostIndexes(redis = requireControlRedis()) { - const hostKeys = await scanKeys(redis, `${HOSTS_PREFIX}*`); - const oldDeclarationKeys = await scanKeys(redis, `${HOST_DECLARATIONS_PREFIX}*`); + const hostKeys = await scanKeys(redis, HOSTS_SCAN_PATTERN); + const oldDeclarationKeys = await scanKeys(redis, HOST_DECLARATIONS_SCAN_PATTERN); /** @type {Map>} */ const declarationsByHost = new Map(); for (const key of hostKeys) { @@ -457,7 +365,7 @@ export async function rebuildDeclaredHostIndexes(redis = requireControlRedis()) multi.del(DECLARED_HOSTS_KEY, ...oldDeclarationKeys); for (const [host, namespaces] of declarationsByHost) { multi.sAdd(DECLARED_HOSTS_KEY, host); - multi.sAdd(`${HOST_DECLARATIONS_PREFIX}${host}`, [...namespaces]); + multi.sAdd(hostDeclarationsKey(host), [...namespaces]); } await multi.exec(); }); @@ -528,23 +436,37 @@ export function buildS3CleanupTaskId() { return `${S3_CLEANUP_TASK_ID_PREFIX}${crypto.randomUUID()}`; } -export function generateLockToken() { - return crypto.randomUUID(); -} - -// TTL only matters when a handler crashes mid-flow (normal path releases -// in a finally block); 30s covers the Redis critical section and no more. +// Normal paths release this advisory lock in finally; the TTL only bounds +// leaks after a crash. Delete transactions also verify this token under their +// final WATCH so an expired request cannot commit under a replacement holder. const DELETE_LOCK_TTL_SECONDS = 30; /** * @param {RedisClient} redis * @param {string} ns * @param {string} worker + * @param {"whole" | "version"} kind */ -export async function acquireDeleteLock(redis, ns, worker) { - const token = generateLockToken(); +export async function acquireDeleteLock(redis, ns, worker, kind) { const key = deleteLockKey(ns, worker); - const reply = await redis.set(key, token, { nx: true, ttl: DELETE_LOCK_TTL_SECONDS }); - return reply === "OK" ? token : null; + const lock = createTokenLock(key); + lock.token = formatDeleteLockToken(kind, lock.token); + return await acquireTokenLock(redis, lock, { ttlSeconds: DELETE_LOCK_TTL_SECONDS }) + ? lock.token + : null; +} + +/** + * @param {RedisClient} redis + * @param {string} ns + * @param {string} worker + * @param {string} token + */ +export async function renewDeleteLock(redis, ns, worker, token) { + return await renewTokenLock( + redis, + { key: deleteLockKey(ns, worker), token }, + DELETE_LOCK_TTL_SECONDS + ); } // Compare-and-delete so we don't free a token a TTL-expired-then-reacquired @@ -559,15 +481,13 @@ export async function acquireDeleteLock(redis, ns, worker) { export async function releaseDeleteLock(redis, ns, worker, token, requestId) { if (!token) return; const key = deleteLockKey(ns, worker); - try { - await redis.delIfEq(key, token); - } catch (err) { - requireControlLog()("warn", "delete_lock_release_failed", { + await releaseTokenLock(redis, { key, token }, { + onError: (err) => requireControlLog()("warn", "delete_lock_release_failed", { request_id: requestId, namespace: ns, worker, ...formatError(err), - }); - } + }), + }); } // Standalone intent write for deploy aborts, where the cleanup follows a @@ -623,54 +543,37 @@ export async function recordCleanupIntentOrWarn({ } /** - * @param {{ ns: string, worker: string, version?: string, allowCleanup?: boolean }} args + * @param {{ ns: string, worker: string, version?: string, allowCleanup?: boolean, requestId?: string | null }} args */ -export async function assertWorkflowDeleteAllowed({ ns, worker, version = undefined, allowCleanup = false }) { - if (!state.workflows || typeof state.workflows.fetch !== "function") { - throw new ControlAbort(503, "workflow_internal_dispatch_failed", { - message: "Workflow lifecycle check is unavailable", - namespace: ns, - worker, - ...(version ? { version } : {}), - }); - } - let response; - let body; - try { - response = await state.workflows.fetch( - "http://workflows/internal/workflows/lifecycle/check-delete", - { - method: "POST", - headers: controlInternalJsonHeaders(), - body: JSON.stringify({ - ns, - worker, - ...(version ? { version } : {}), - ...(allowCleanup ? { allowCleanup: true } : {}), - }), - }, - ); - body = await response.json().catch(() => null); - } catch (err) { - state.log?.("error", "workflow_lifecycle_check_failed", { - namespace: ns, - worker, - ...(version ? { version } : {}), - error_message: errMessage(err), - }); - throw new ControlAbort(503, "workflow_internal_dispatch_failed", { - message: "Workflow lifecycle check failed", - namespace: ns, +export async function assertWorkflowDeleteAllowed({ ns, worker, version = undefined, allowCleanup = false, requestId = null }) { + const context = { + namespace: ns, + worker, + ...(version ? { version } : {}), + }; + const { response, body } = await postWorkflowsInternal({ + endpoint: "workflows/lifecycle/check-delete", + body: { + ns, worker, ...(version ? { version } : {}), - }); - } + ...(allowCleanup ? { allowCleanup: true } : {}), + }, + requestId, + logEvent: "workflow_lifecycle_check_failed", + logFields: context, + errorDetails: context, + unavailableMessage: "Workflow lifecycle check is unavailable", + requestFailedMessage: "Workflow lifecycle check failed", + // The lifecycle scan is unbounded by namespace size. Preserve its + // pre-consolidation behavior instead of imposing the ordinary short + // control-to-workflows request timeout. + timeoutMs: null, + }); if (!response.ok) { throw new ControlAbort(503, "workflow_internal_dispatch_failed", { message: "Workflow lifecycle check failed", - namespace: ns, - worker, - ...(version ? { version } : {}), + ...context, upstream_status: response.status, upstream_error: isRecord(body) && typeof body.error === "string" ? body.error : null, }); @@ -678,9 +581,7 @@ export async function assertWorkflowDeleteAllowed({ ns, worker, version = undefi if (!isRecord(body) || typeof body.allowed !== "boolean") { throw new ControlAbort(503, "workflow_internal_dispatch_failed", { message: "Workflow lifecycle check returned an invalid response", - namespace: ns, - worker, - ...(version ? { version } : {}), + ...context, }); } if (body.allowed !== true) { @@ -688,9 +589,7 @@ export async function assertWorkflowDeleteAllowed({ ns, worker, version = undefi message: version ? `${ns}/${worker}/${version} has active workflow instances` : `${ns}/${worker} has active workflow instances`, - namespace: ns, - worker, - ...(version ? { version } : {}), + ...context, count: Number.isFinite(body.count) ? body.count : 0, blockers: Array.isArray(body.blockers) ? body.blockers : [], }); @@ -698,46 +597,26 @@ export async function assertWorkflowDeleteAllowed({ ns, worker, version = undefi } /** - * @param {{ ns: string, worker: string, doStorageId: string }} args + * @param {{ ns: string, worker: string, doStorageId: string, requestId?: string | null }} args */ -export async function cleanupDoAlarmsForWorker({ ns, worker, doStorageId }) { - if (!state.workflows || typeof state.workflows.fetch !== "function") { - throw new ControlAbort(503, "workflow_internal_dispatch_failed", { - message: "Workflow DO alarm cleanup is unavailable", - namespace: ns, - worker, - }); - } - let response; - let body; - try { - response = await state.workflows.fetch( - "http://workflows/internal/workflows/do-alarms/cleanup-worker", - { - method: "POST", - headers: controlInternalJsonHeaders(), - body: JSON.stringify({ ns, worker, doStorageId }), - signal: AbortSignal.timeout(DO_ALARM_CLEANUP_TIMEOUT_MS), - }, - ); - body = await response.json().catch(() => null); - } catch (err) { - state.log?.("error", "workflow_do_alarm_cleanup_failed", { - namespace: ns, - worker, - error_message: errMessage(err), - }); - throw new ControlAbort(503, "workflow_internal_dispatch_failed", { - message: "Workflow DO alarm cleanup failed", - namespace: ns, - worker, - }); - } +export async function cleanupDoAlarmsForWorker({ ns, worker, doStorageId, requestId = null }) { + const context = { namespace: ns, worker }; + const { response, body } = await postWorkflowsInternal({ + endpoint: "workflows/do-alarms/cleanup-worker", + body: { ns, worker, doStorageId }, + requestId, + logEvent: "workflow_do_alarm_cleanup_failed", + logFields: context, + errorDetails: context, + unavailableMessage: "Workflow DO alarm cleanup is unavailable", + requestFailedMessage: "Workflow DO alarm cleanup failed", + // This endpoint was already bounded before transport consolidation. + timeoutMs: WORKFLOWS_INTERNAL_TIMEOUT_MS, + }); if (!response.ok || !isRecord(body) || body.ok !== true) { throw new ControlAbort(503, "workflow_internal_dispatch_failed", { message: "Workflow DO alarm cleanup failed", - namespace: ns, - worker, + ...context, upstream_status: response.status, upstream_error: isRecord(body) && typeof body.error === "string" ? body.error : null, }); diff --git a/control/workflows-client.js b/control/workflows-client.js new file mode 100644 index 0000000..717ab50 --- /dev/null +++ b/control/workflows-client.js @@ -0,0 +1,118 @@ +import { errorMessage } from "shared-errors"; +import { ControlAbort } from "control-errors"; + +export const WORKFLOWS_INTERNAL_TIMEOUT_MS = 5_000; + +/** + * @typedef {{ fetch: typeof fetch }} WorkflowBackend + * @typedef {(level: string, event: string, fields?: Record) => void} WorkflowClientLogger + * @typedef {"unavailable" | "request_failed"} WorkflowTransportFailure + */ + +/** + * @param {{ + * workflows: WorkflowBackend | null | undefined, + * headers: () => HeadersInit, + * endpoint: string, + * body: unknown, + * requestId?: string | null, + * log?: WorkflowClientLogger | null, + * logEvent: string, + * logFields?: Record, + * timeoutMs: number | null, + * makeError: (failure: WorkflowTransportFailure) => Error, + * }} args + * @returns {Promise<{ response: Response, body: unknown }>} + */ +export async function postWorkflowsInternalRequest({ + workflows, + headers, + endpoint, + body, + requestId = null, + log = null, + logEvent, + logFields = {}, + timeoutMs, + makeError, +}) { + if (!workflows || typeof workflows.fetch !== "function") { + throw makeError("unavailable"); + } + + try { + const requestHeaders = new Headers(headers()); + if (typeof requestId === "string" && requestId) { + requestHeaders.set("x-request-id", requestId); + } + const response = await workflows.fetch(`http://workflows/internal/${endpoint}`, { + method: "POST", + headers: requestHeaders, + body: JSON.stringify(body), + ...(timeoutMs === null ? {} : { signal: AbortSignal.timeout(timeoutMs) }), + }); + return { + response, + body: await response.json().catch(() => null), + }; + } catch (err) { + log?.("error", logEvent, { + ...logFields, + ...(typeof requestId === "string" && requestId ? { request_id: requestId } : {}), + error_message: errorMessage(err), + }); + throw makeError("request_failed"); + } +} + +/** + * @param {{ + * getWorkflows: () => WorkflowBackend | null | undefined, + * headers: () => HeadersInit, + * getLog?: () => WorkflowClientLogger | null | undefined, + * }} dependencies + */ +export function createPostWorkflowsInternal({ getWorkflows, headers, getLog = () => null }) { + /** + * @param {{ + * endpoint: string, + * body: unknown, + * requestId?: string | null, + * logEvent: string, + * logFields?: Record, + * errorDetails?: Record, + * timeoutMs: number | null, + * unavailableMessage?: string, + * requestFailedMessage?: string, + * }} args + */ + return async function postWorkflowsInternal({ + endpoint, + body, + requestId = null, + logEvent, + logFields = {}, + errorDetails = {}, + timeoutMs, + unavailableMessage = "Workflow backend is unavailable", + requestFailedMessage = "Workflow backend request failed", + }) { + return await postWorkflowsInternalRequest({ + workflows: getWorkflows(), + headers, + endpoint, + body, + requestId, + log: getLog(), + logEvent, + logFields, + timeoutMs, + makeError: (failure) => new ControlAbort(503, "workflow_internal_dispatch_failed", { + ...errorDetails, + message: failure === "unavailable" + ? unavailableMessage + : requestFailedMessage, + }), + }); + }; +} diff --git a/d1-runtime/config.capnp b/d1-runtime/config.capnp index 420ab67..820714b 100644 --- a/d1-runtime/config.capnp +++ b/d1-runtime/config.capnp @@ -44,8 +44,11 @@ const d1RuntimeWorker :Workerd.Worker = ( (name = "shared-request-scope", esModule = embed "../shared/request-scope.js"), (name = "shared-env", esModule = embed "../shared/env.js"), (name = "shared-fnv1a32", esModule = embed "../shared/fnv1a32.js"), + (name = "shared-ns-pattern", esModule = embed "../shared/ns-pattern.js"), (name = "shared-task-identity", esModule = embed "../shared/task-identity.js"), + (name = "shared-owner-endpoint", esModule = embed "../shared/owner-endpoint.js"), (name = "shared-owner-lease", esModule = embed "../shared/owner-lease.js"), + (name = "shared-optimistic-retry", esModule = embed "../shared/optimistic-retry.js"), (name = "shared-owner-protocol", esModule = embed "../shared/owner-protocol.js"), (name = "shared-owner-forwarder", esModule = embed "../shared/owner-forwarder.js"), (name = "shared-redis-client", esModule = embed "../shared/redis-client.js"), @@ -71,7 +74,6 @@ const d1RuntimeWorker :Workerd.Worker = ( (name = "D1_TASK_ID", fromEnvironment = "D1_TASK_ID"), (name = "D1_TASK_ENDPOINT", fromEnvironment = "D1_TASK_ENDPOINT"), (name = "D1_TASK_CONTAINER_NAME", fromEnvironment = "D1_TASK_CONTAINER_NAME"), - (name = "D1_TASK_PORT", fromEnvironment = "D1_TASK_PORT"), (name = "D1_TASK_IDENTITY_TIMEOUT_MS", fromEnvironment = "D1_TASK_IDENTITY_TIMEOUT_MS"), (name = "ECS_CONTAINER_METADATA_URI_V4", fromEnvironment = "ECS_CONTAINER_METADATA_URI_V4"), (name = "D1_OWNER_TTL_SECONDS", fromEnvironment = "D1_OWNER_TTL_SECONDS"), diff --git a/d1-runtime/owner-client.js b/d1-runtime/owner-client.js index e0bba63..b74a21a 100644 --- a/d1-runtime/owner-client.js +++ b/d1-runtime/owner-client.js @@ -21,6 +21,7 @@ import { } from "d1-runtime-state"; import { withInternalAuth } from "shared-internal-auth"; import { forwardOwnerRequest } from "shared-owner-forwarder"; +import { validOwnerEndpointForService } from "shared-owner-endpoint"; /** * @typedef {Record & { D1_QUERY_TIMEOUT_MS?: unknown }} D1Env @@ -30,6 +31,7 @@ import { forwardOwnerRequest } from "shared-owner-forwarder"; */ const OWNER_UNAVAILABLE_STATUSES = new Set([502, 503, 504]); +const D1_OWNER_PORT = 8787; /** @param {Headers} headers */ function isD1QueryResponse(headers) { @@ -53,6 +55,9 @@ function resultUnknownError(query, detail) { /** @param {D1Env} env @param {D1Query} query @param {D1Owner} owner */ export async function probeOwner(env, query, owner) { if (!owner.endpoint) return { outcome: "owner-endpoint-missing" }; + if (!validOwnerEndpointForService(owner.endpoint, D1_OWNER_PORT, "d1-runtime")) { + return { outcome: "owner-endpoint-invalid" }; + } try { const res = await fetch( `http://${owner.endpoint}/internal/d1/probe?dbKey=${encodeURIComponent(query.dbKey)}&generation=${owner.generation}`, @@ -83,6 +88,8 @@ export async function forwardToOwner(query, env, owner, requestId = null, hopCou const response = await forwardOwnerRequest({ env, endpoint: owner.endpoint, + endpointPort: D1_OWNER_PORT, + endpointService: "d1-runtime", pathname: "/internal/d1/query", requestId, hopCount, @@ -113,6 +120,8 @@ export async function forwardToOwner(query, env, owner, requestId = null, hopCou }), missingEndpointError: () => new D1ProtocolError(503, "owner-endpoint-missing", `D1 database ${query.dbKey} owner has no endpoint`), + invalidEndpointError: () => + new D1ProtocolError(503, "owner-endpoint-invalid", `D1 database ${query.dbKey} owner endpoint is invalid`), hopExhaustedError: () => new D1ProtocolError( 503, diff --git a/d1-runtime/owner-registry.js b/d1-runtime/owner-registry.js index 795d527..195d2f1 100644 --- a/d1-runtime/owner-registry.js +++ b/d1-runtime/owner-registry.js @@ -10,6 +10,7 @@ import { decodeBulk, WatchError } from "shared-redis"; import { envValueOr } from "shared-env"; import { errorMessage } from "shared-errors"; import { createRequiredRedisClient } from "shared-redis-client"; +import { validOwnerEndpointForService } from "shared-owner-endpoint"; import { boundedPositiveIntEnv, currentOwnerGenerationCounter, @@ -55,6 +56,7 @@ const DRAIN_WAIT_POLL_MS = 25; const OWNER_PREFIX = "d1:owner:db:"; const OWNER_CLAIM_RETRIES = 3; const OWNER_CLAIM_WINNER_READ_DELAYS_MS = [0, 10, 25, 50, 100]; +const D1_OWNER_PORT = 8787; /** * @typedef {{ idFromName(name: string): unknown, get(id: unknown): { fetch(url: string, init?: RequestInit): Promise } }} D1Namespace @@ -127,9 +129,39 @@ export function ownerGenerationKeyOf(dbKey) { return ownerProtocolKeys(OWNER_PREFIX, dbKey).generationKey; } -/** @param {string | null | undefined} raw @returns {D1Owner | null} */ -export function parseOwner(raw) { - return /** @type {D1Owner | null} */ (parseOwnerRecord(raw)); +/** + * @param {D1Owner | null} owner + * @param {string} expectedDbKey + */ +function ownerMatchesScope(owner, expectedDbKey) { + if ( + !owner || + typeof owner.namespace !== "string" || + typeof owner.databaseId !== "string" || + typeof owner.dbKey !== "string" || + typeof owner.slot !== "number" || + !Number.isSafeInteger(owner.slot) || owner.slot < 0 + ) return false; + try { + return owner.dbKey === expectedDbKey && dbKeyOf(owner.namespace, owner.databaseId) === expectedDbKey; + } catch { + return false; + } +} + +/** @param {string | null | undefined} raw @param {string} expectedDbKey @returns {D1Owner | null} */ +export function parseOwner(raw, expectedDbKey) { + if (raw == null) return null; + const owner = /** @type {D1Owner | null} */ (parseOwnerRecord(raw)); + if ( + !owner || + !ownerMatchesScope(owner, expectedDbKey) || + typeof owner.taskId !== "string" || !owner.taskId || + !validOwnerEndpointForService(owner.endpoint, D1_OWNER_PORT, "d1-runtime") + ) { + throw new D1ProtocolError(503, "owner-record-invalid", "D1 owner record is invalid"); + } + return owner; } /** @param {string} dbKey @returns {never} */ @@ -148,12 +180,12 @@ export async function readOwner(env, dbKey) { /** @param {RedisClient} client @param {string} dbKey */ async function readOwnerFromClient(client, dbKey) { - return await readOwnerRecord(client, ownerKeyOf(dbKey), (raw) => parseOwner(decodeBulk(raw))); + return await readOwnerRecord(client, ownerKeyOf(dbKey), (raw) => parseOwner(decodeBulk(raw), dbKey)); } /** @param {{ getWithTime(key: string): Promise<{ value: string | Uint8Array | null | undefined, nowMs: number }> }} client @param {string} dbKey */ async function readOwnerWithTimeFromClient(client, dbKey) { - return await readOwnerRecordWithRedisTime(client, ownerKeyOf(dbKey), (raw) => parseOwner(decodeBulk(raw))); + return await readOwnerRecordWithRedisTime(client, ownerKeyOf(dbKey), (raw) => parseOwner(decodeBulk(raw), dbKey)); } /** @param {RedisClient} client @param {string} dbKey */ @@ -168,6 +200,9 @@ async function readOwnerWithTimeAfterClaimRace(client, dbKey) { /** @param {D1Env} env @param {D1Identity} identity @param {number} generation @param {D1Target} target @param {number} nowMs */ function ownerRecordFor(env, identity, generation, target, nowMs) { + if (!validOwnerEndpointForService(target.endpoint, D1_OWNER_PORT, "d1-runtime")) { + throw new D1ProtocolError(503, "owner-record-invalid", "D1 owner target endpoint is invalid"); + } const ttl = ownerTtlSeconds(env); return { namespace: identity.namespace, @@ -465,7 +500,7 @@ export async function releaseOwner(env, owner) { const key = ownerKeyOf(owner.dbKey); return await withWatchRetries(async () => client.session(async (session) => { await session.watch(key); - const current = parseOwner(await session.get(key)); + const current = parseOwner(await session.get(key), owner.dbKey); if (!ownerFenceMatches(current, owner)) { await session.unwatch(); forgetOwnedDb(owner.dbKey); @@ -645,9 +680,6 @@ export function normalizeDatabases(databases) { const record = /** @type {Record} */ (database); const namespace = record.namespace; const databaseId = record.databaseId; - if (typeof databaseId === "string" && databaseId.includes(":")) { - throw new D1ProtocolError(400, "invalid-database-id", "databaseId must not contain ':'"); - } const dbKey = dbKeyOf(namespace, databaseId); const normalizedNamespace = /** @type {string} */ (namespace); const normalizedDatabaseId = /** @type {string} */ (databaseId); @@ -673,6 +705,9 @@ export function normalizeTarget(target) { if (typeof record.endpoint !== "string" || !record.endpoint) { throw new D1ProtocolError(400, "invalid-target", "target.endpoint is required"); } + if (!validOwnerEndpointForService(record.endpoint, D1_OWNER_PORT, "d1-runtime")) { + throw new D1ProtocolError(400, "invalid-target", "target.endpoint is invalid"); + } return { taskId: record.taskId, endpoint: record.endpoint }; } @@ -684,7 +719,7 @@ export async function rebalanceDatabase(env, database, target) { const localTask = await resolveTaskIdentity(env); return await withWatchRetries(async () => client.session(async (session) => { await session.watch(key, generationKey); - const current = parseOwner(await session.get(key)); + const current = parseOwner(await session.get(key), database.dbKey); if (!current || current.taskId !== localTask.taskId) { await session.unwatch(); forgetOwnedDb(database.dbKey); diff --git a/d1-runtime/protocol.js b/d1-runtime/protocol.js index cb1e059..4dbb5d2 100644 --- a/d1-runtime/protocol.js +++ b/d1-runtime/protocol.js @@ -2,6 +2,7 @@ import { normalizeD1Param } from "shared-d1-params"; import { fnv1a32CodeUnits } from "shared-fnv1a32"; import { BodyTooLargeError, readBoundedBytes as readRequestBoundedBytes } from "shared-bounded-body"; import { errorMessage as sharedErrorMessage } from "shared-errors"; +import { D1_DATABASE_ID_RE, isValidRuntimeLoadNs } from "shared-ns-pattern"; import { D1_QUERY_CONTENT_TYPE, D1_QUERY_RESPONSE_CONTENT_TYPE, @@ -96,11 +97,11 @@ export class D1ProtocolError extends Error { * @param {unknown} databaseId */ export function dbKeyOf(namespace, databaseId) { - if (typeof namespace !== "string" || !namespace) { - throw new D1ProtocolError(400, "invalid-namespace", "namespace is required"); + if (!isValidRuntimeLoadNs(namespace)) { + throw new D1ProtocolError(400, "invalid-namespace", "namespace is invalid"); } - if (typeof databaseId !== "string" || !databaseId) { - throw new D1ProtocolError(400, "invalid-database-id", "databaseId is required"); + if (typeof databaseId !== "string" || !D1_DATABASE_ID_RE.test(databaseId)) { + throw new D1ProtocolError(400, "invalid-database-id", "databaseId is invalid"); } return `${namespace}:${databaseId}`; } @@ -365,7 +366,9 @@ const OWNERSHIP_CODES = new Set([ "not-owner", "owner-not-ready", "owner-unavailable", + "owner-record-invalid", "owner-endpoint-missing", + "owner-endpoint-invalid", "forward-hop-exhausted", "owner-claim-raced", "owner-takeover-raced", diff --git a/d1-runtime/task-identity.js b/d1-runtime/task-identity.js index b7590db..af2b554 100644 --- a/d1-runtime/task-identity.js +++ b/d1-runtime/task-identity.js @@ -1,4 +1,5 @@ import { createTaskIdentityResolver } from "shared-task-identity"; +import { validOwnerEndpointForService } from "shared-owner-endpoint"; import { D1ProtocolError } from "d1-runtime-protocol"; const resolver = createTaskIdentityResolver({ @@ -8,6 +9,7 @@ const resolver = createTaskIdentityResolver({ serviceLabel: "D1", unavailableCode: "task-identity-unavailable", createError: (status, code, message) => new D1ProtocolError(status, code, message), + validateEndpoint: (endpoint, port) => validOwnerEndpointForService(endpoint, port, "d1-runtime"), }); export const { diff --git a/deploy/kubernetes/README.md b/deploy/kubernetes/README.md index 1c68276..121bbe0 100644 --- a/deploy/kubernetes/README.md +++ b/deploy/kubernetes/README.md @@ -13,13 +13,14 @@ provider's native tooling. ## Layout - `base/` defines reusable Deployments, StatefulSets, Services, and PVCs. +- `components/metadata-guard/` owns the shared cloud-metadata egress policy. - `overlays/local/` pins local images, local development secrets, a namespace, and local storage defaults. - `overlays/local-ingress/` adds nginx Ingress resources for local hostnames. -- `overlays/local-metadata-guard/` applies the local overlay plus a minimal - metadata egress guard. +- `overlays/local-metadata-guard/` applies the local overlay plus the shared + metadata-guard component. - `overlays/local-ingress-metadata-guard/` applies the local ingress overlay - plus the same metadata egress guard. + plus the same component. - `storage/local-nfs/` contains Helm values for a local ReadWriteMany provisioner that can simulate provider RWX storage on a development cluster. diff --git a/deploy/kubernetes/README.zh.md b/deploy/kubernetes/README.zh.md index e836580..e1ff896 100644 --- a/deploy/kubernetes/README.zh.md +++ b/deploy/kubernetes/README.zh.md @@ -7,10 +7,11 @@ ## 目录结构 - `base/` 定义可复用的 Deployment、StatefulSet、Service 和 PVC。 +- `components/metadata-guard/` 统一维护云 metadata egress policy。 - `overlays/local/` 固定本地镜像、本地开发 secret、namespace 和本地存储默认值。 - `overlays/local-ingress/` 为本地主机名增加 nginx Ingress 资源。 -- `overlays/local-metadata-guard/` 应用本地 overlay,并额外加入一个最小 metadata egress guard。 -- `overlays/local-ingress-metadata-guard/` 应用本地 ingress overlay,并额外加入同一条 metadata egress guard。 +- `overlays/local-metadata-guard/` 应用本地 overlay,并额外加入共享的 metadata-guard component。 +- `overlays/local-ingress-metadata-guard/` 应用本地 ingress overlay,并额外加入同一个 component。 - `storage/local-nfs/` 包含一个本地 ReadWriteMany provisioner 的 Helm values,可在开发集群中模拟云厂商 RWX 存储。 base 使用 production-style 镜像契约:workerd 服务默认使用 `docker.io/getwdl/wdl-workerd:latest`,Rust sidecar 和 Rust service 默认使用 `docker.io/getwdl/wdl-rust:latest`。本地 overlay 会把这些 public image name 映射到 Compose 集成测试本地构建的 `wdl-workerd:dev` 和 `wdl-rust:dev` tag。 diff --git a/deploy/kubernetes/base/network-policy.yaml b/deploy/kubernetes/base/network-policy.yaml index 8cf4981..ea002c9 100644 --- a/deploy/kubernetes/base/network-policy.yaml +++ b/deploy/kubernetes/base/network-policy.yaml @@ -50,10 +50,6 @@ spec: ports: - port: 8081 - from: - - podSelector: - matchLabels: - app.kubernetes.io/name: wdl - app.kubernetes.io/component: system-runtime - podSelector: matchLabels: app.kubernetes.io/name: wdl diff --git a/deploy/kubernetes/base/scheduler.yaml b/deploy/kubernetes/base/scheduler.yaml index 2bd97e9..b8dc59d 100644 --- a/deploy/kubernetes/base/scheduler.yaml +++ b/deploy/kubernetes/base/scheduler.yaml @@ -19,6 +19,7 @@ spec: app.kubernetes.io/name: wdl app.kubernetes.io/component: scheduler spec: + terminationGracePeriodSeconds: 30 containers: - name: scheduler image: docker.io/getwdl/wdl-rust:latest diff --git a/deploy/kubernetes/base/workflows.yaml b/deploy/kubernetes/base/workflows.yaml index 955e3f9..5927d84 100644 --- a/deploy/kubernetes/base/workflows.yaml +++ b/deploy/kubernetes/base/workflows.yaml @@ -33,6 +33,7 @@ spec: app.kubernetes.io/name: wdl app.kubernetes.io/component: workflows spec: + terminationGracePeriodSeconds: 30 containers: - name: workflows image: docker.io/getwdl/wdl-rust:latest diff --git a/deploy/kubernetes/components/metadata-guard/kustomization.yaml b/deploy/kubernetes/components/metadata-guard/kustomization.yaml new file mode 100644 index 0000000..7fff9b7 --- /dev/null +++ b/deploy/kubernetes/components/metadata-guard/kustomization.yaml @@ -0,0 +1,4 @@ +apiVersion: kustomize.config.k8s.io/v1alpha1 +kind: Component +resources: + - metadata-egress-policy.yaml diff --git a/deploy/kubernetes/overlays/local-ingress-metadata-guard/metadata-egress-policy.yaml b/deploy/kubernetes/components/metadata-guard/metadata-egress-policy.yaml similarity index 100% rename from deploy/kubernetes/overlays/local-ingress-metadata-guard/metadata-egress-policy.yaml rename to deploy/kubernetes/components/metadata-guard/metadata-egress-policy.yaml diff --git a/deploy/kubernetes/overlays/local-ingress-metadata-guard/kustomization.yaml b/deploy/kubernetes/overlays/local-ingress-metadata-guard/kustomization.yaml index 7efb7da..5830040 100644 --- a/deploy/kubernetes/overlays/local-ingress-metadata-guard/kustomization.yaml +++ b/deploy/kubernetes/overlays/local-ingress-metadata-guard/kustomization.yaml @@ -3,4 +3,5 @@ kind: Kustomization namespace: wdl-local resources: - ../local-ingress - - metadata-egress-policy.yaml +components: + - ../../components/metadata-guard diff --git a/deploy/kubernetes/overlays/local-metadata-guard/kustomization.yaml b/deploy/kubernetes/overlays/local-metadata-guard/kustomization.yaml index ab73d19..a8ee06e 100644 --- a/deploy/kubernetes/overlays/local-metadata-guard/kustomization.yaml +++ b/deploy/kubernetes/overlays/local-metadata-guard/kustomization.yaml @@ -3,4 +3,5 @@ kind: Kustomization namespace: wdl-local resources: - ../local - - metadata-egress-policy.yaml +components: + - ../../components/metadata-guard diff --git a/deploy/kubernetes/overlays/local-metadata-guard/metadata-egress-policy.yaml b/deploy/kubernetes/overlays/local-metadata-guard/metadata-egress-policy.yaml deleted file mode 100644 index e28b744..0000000 --- a/deploy/kubernetes/overlays/local-metadata-guard/metadata-egress-policy.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: NetworkPolicy -metadata: - name: block-cloud-metadata-egress - labels: - app.kubernetes.io/name: wdl - app.kubernetes.io/component: network-policy -spec: - podSelector: - matchLabels: - app.kubernetes.io/name: wdl - policyTypes: - - Egress - egress: - - to: - - ipBlock: - cidr: 0.0.0.0/0 - except: - - 169.254.169.254/32 - - 169.254.170.2/32 diff --git a/do-runtime/actor.js b/do-runtime/actor.js index 9a9f8a3..8f56371 100644 --- a/do-runtime/actor.js +++ b/do-runtime/actor.js @@ -5,7 +5,10 @@ import { buildAlarmRequest, buildFacetName, buildForwardRequest, - doErrorResponse, + buildRpcRequest, + DO_OWNERSHIP_ERROR_CONTROL_HEADER, + DO_OWNERSHIP_CODE, + doPlatformErrorResponse, DoRuntimeError, normalizeDoConnectRequest, readLocalActorInvokeRequest, @@ -23,11 +26,11 @@ import { metrics, SERVICE, } from "do-runtime-state"; -import { errorMessage } from "shared-errors"; import { formatError } from "shared-observability"; +import { rebuildResponseWithHeaders } from "shared-respond"; /** - * @typedef {{ LOADER: { get(key: string, loader: () => Promise): DoWorkerStub }, DO_TEST_HOOKS?: unknown }} DoEnv + * @typedef {{ LOADER: { get(key: string, loader: () => Promise): DoWorkerStub } }} DoEnv * @typedef {{ getDurableObjectClass(className: string, options: { props: Record }): DurableObjectClass }} DoWorkerStub * @typedef {{ fetch(request: Request): Promise }} DoFacet * @typedef {import("do-runtime-protocol").DoInvoke} DoInvoke @@ -63,12 +66,11 @@ export class WdlDoHostActor extends DurableObject { tenantWorker(invoke, requestId) { const existing = this.workers.get(invoke.workerId); if (existing) return existing; - const workerCode = "workerCode" in invoke ? invoke.workerCode : undefined; const worker = this.env.LOADER.get(invoke.workerId, () => ( - workerCode || loadDoWorkerCode( + loadDoWorkerCode( this.env, this.ctx, - /** @type {DoInvoke & { ns: string, worker: string, version: string, doStorageId: string }} */ (invoke), + invoke, requestId ) )); @@ -133,9 +135,7 @@ export class WdlDoHostActor extends DurableObject { }); } if (url.pathname === "/delete-storage") { - const invoke = /** @type {DoInvoke} */ (await readLocalActorInvokeRequest(request, { - allowInlineWorkerCode: false, - })); + const invoke = /** @type {DoInvoke} */ (await readLocalActorInvokeRequest(request)); await assertCurrentOwner(this.env, invoke.owner); const facetName = buildFacetName(invoke); this.ctx.facets.delete(facetName); @@ -145,9 +145,7 @@ export class WdlDoHostActor extends DurableObject { metrics.setGauge("do_host_actor_object_registry_size", { service: SERVICE }, this.registeredObjectMembers.size); return Response.json({ ok: true }); } - const invoke = /** @type {DoInvoke} */ (await readLocalActorInvokeRequest(request, { - allowInlineWorkerCode: this.env.DO_TEST_HOOKS === "1", - })); + const invoke = /** @type {DoInvoke} */ (await readLocalActorInvokeRequest(request)); return await this.dispatchWithFence(invoke, async () => { const requestId = request.headers.get("x-request-id") || null; const cls = this.tenantWorker(invoke, requestId).getDurableObjectClass(invoke.className, { @@ -158,15 +156,15 @@ export class WdlDoHostActor extends DurableObject { id: invoke.objectName, })); if (invoke.kind === "alarm") { - return await facet.fetch(buildAlarmRequest(invoke.alarm)); + return await facet.fetch(buildAlarmRequest(invoke.alarm, requestId)); } if (invoke.kind === "rpc") { - return await dispatchRpc(facet, invoke.rpc); + return await dispatchRpc(facet, invoke.rpc, requestId); } return await facet.fetch(buildForwardRequest(invoke.request)); }); } catch (err) { - return doErrorResponse(err); + return doPlatformErrorResponse(err); } } @@ -176,12 +174,14 @@ export class WdlDoHostActor extends DurableObject { */ async dispatchWithFence(invoke, run) { if (!beginInFlightDispatch()) { - throw new DoRuntimeError(503, "task_draining", "DO task is draining"); + throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.TASK_DRAINING, "DO task is draining"); } try { - const { owner, leaseRemainingMs } = await assertCurrentOwnerWithLeaseBudget(this.env, invoke.owner); await this.rememberObject(invoke); - return await this.dispatchWithLeaseBudget(invoke, owner, leaseRemainingMs, run); + const { owner, leaseRemainingMs } = await assertCurrentOwnerWithLeaseBudget(this.env, invoke.owner); + return withoutOwnershipErrorControlHeader( + await this.dispatchWithLeaseBudget(invoke, owner, leaseRemainingMs, run) + ); } finally { endInFlightDispatch(); } @@ -243,9 +243,9 @@ export class WdlDoHostActor extends DurableObject { if (!schedule(leaseRemainingMs)) { if (scheduleFailureReason === "lease_guard") { - throw new DoRuntimeError(503, "owner_lease_too_short", `DO scope ${owner.ownerKey} owner lease has insufficient remaining budget`); + throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_LEASE_TOO_SHORT, `DO scope ${owner.ownerKey} owner lease has insufficient remaining budget`); } - throw new DoRuntimeError(503, "owner_lease_expired", `DO scope ${owner.ownerKey} owner lease has expired`); + throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_LEASE_EXPIRED, `DO scope ${owner.ownerKey} owner lease has expired`); } try { return await run(); @@ -256,34 +256,19 @@ export class WdlDoHostActor extends DurableObject { } } +/** @param {Response} response */ +function withoutOwnershipErrorControlHeader(response) { + if (!response.headers.has(DO_OWNERSHIP_ERROR_CONTROL_HEADER)) return response; + const headers = new Headers(response.headers); + headers.delete(DO_OWNERSHIP_ERROR_CONTROL_HEADER); + return rebuildResponseWithHeaders(response, headers); +} + /** * @param {DoFacet} facet * @param {{ method: string, args: unknown[] }} rpc + * @param {string | null} requestId */ -async function dispatchRpc(facet, rpc) { - // Facets are workerd JSRPC stubs; reading a method returns a forwarder - // already bound to that stub, unlike ordinary unbound JavaScript methods. - const methods = /** @type {Record} */ (/** @type {unknown} */ (facet)); - const fn = methods[rpc.method]; - if (typeof fn !== "function") { - return Response.json({ - error: "do_rpc_method_not_found", - message: `Durable Object RPC method ${rpc.method} was not found`, - }, { status: 404 }); - } - try { - return Response.json({ ok: true, result: await fn(...rpc.args) }); - } catch (err) { - const errorObject = err && typeof err === "object" ? err : {}; - // The stack captured from fn(...) is tenant method execution, not - // do-runtime framework internals, so it belongs on the tenant RPC boundary. - // Runtime failures still go through doErrorResponse(), which intentionally - // hides internal stack traces and infrastructure details. - return Response.json({ - error: "do_rpc_error", - name: "name" in errorObject && typeof errorObject.name === "string" ? errorObject.name : "Error", - message: errorMessage(err), - ...("stack" in errorObject && typeof errorObject.stack === "string" && errorObject.stack ? { stack: errorObject.stack } : {}), - }, { status: 500 }); - } +export async function dispatchRpc(facet, rpc, requestId) { + return await facet.fetch(buildRpcRequest(rpc, requestId)); } diff --git a/do-runtime/alarm-dispatch.js b/do-runtime/alarm-dispatch.js index 28809e1..0beadfa 100644 --- a/do-runtime/alarm-dispatch.js +++ b/do-runtime/alarm-dispatch.js @@ -1,5 +1,4 @@ -import { formatWorkerId } from "shared-worker-id"; -import { DoRuntimeError, hostIdForObject, nonEmptyAlarmString, readJsonBody } from "do-runtime-protocol"; +import { DoRuntimeError, normalizeDoInvokeRequest, readJsonBody } from "do-runtime-protocol"; import { json } from "do-runtime-http"; /** @@ -8,26 +7,6 @@ import { json } from "do-runtime-http"; * @typedef {(env: DoEnv, invoke: DoInvoke, requestId?: string | null) => Promise} AlarmDispatcher */ -/** - * @param {unknown} value - * @param {string} field - */ -function requiredAlarmString(value, field) { - const text = nonEmptyAlarmString(value); - if (text === null) { - throw new DoRuntimeError(400, "invalid_do_alarm_dispatch", `DO alarm ${field} must be a non-empty string`); - } - return text; -} - -/** @param {unknown} value */ -function alarmRetryCount(value) { - if (typeof value !== "number" || !Number.isInteger(value) || value < 0) { - throw new DoRuntimeError(400, "invalid_do_alarm_dispatch", "DO alarm retryCount must be a non-negative integer"); - } - return value; -} - /** * @param {Request} request * @param {DoEnv} env @@ -39,38 +18,25 @@ export async function handleAlarmDispatch(request, env, dispatchInvoke, requestI const record = body && typeof body === "object" && !Array.isArray(body) ? /** @type {Record} */ (body) : {}; - const ns = requiredAlarmString(record.ns, "ns"); - const worker = requiredAlarmString(record.worker, "worker"); - const version = requiredAlarmString(record.version, "version"); - const doStorageId = requiredAlarmString(record.doStorageId, "doStorageId"); - const className = requiredAlarmString(record.className, "className"); - const objectName = requiredAlarmString(record.objectName, "objectName"); - const retryCount = alarmRetryCount(record.retryCount); - const token = requiredAlarmString(record.token, "token"); - /** @type {DoInvoke} */ - const invoke = { + if (record.retryCount == null) { + throw new DoRuntimeError(400, "invalid_request", "alarm.retryCount is required"); + } + const invoke = normalizeDoInvokeRequest({ kind: "alarm", - ns, - worker, - version, - doStorageId, - workerId: formatWorkerId({ namespace: ns, worker, version }), - hostId: hostIdForObject(doStorageId, className, objectName), - className, - objectName, - props: { - ns, - worker, - version, - doStorageId, - className, - }, + ns: record.ns, + worker: record.worker, + version: record.version, + doStorageId: record.doStorageId, + className: record.className, + objectName: record.objectName, alarm: { - retryCount, - isRetry: retryCount > 0, - token, + retryCount: record.retryCount, + token: record.token, }, - }; + }); + if (invoke.kind !== "alarm" || invoke.alarm.token == null) { + throw new DoRuntimeError(400, "invalid_request", "alarm.token is required"); + } const response = await dispatchInvoke(env, invoke, requestId); const text = await response.text(); if (!response.ok) { diff --git a/do-runtime/alarm-shim-source.js b/do-runtime/alarm-shim-source.js index 15b93b1..9a78989 100644 --- a/do-runtime/alarm-shim-source.js +++ b/do-runtime/alarm-shim-source.js @@ -1,8 +1,41 @@ export const DO_ALARM_SHIM_SOURCE = ` const ALARM_HEADER = "x-wdl-do-internal-alarm"; +const RPC_HEADER = "x-wdl-do-internal-rpc"; const ALARMS_BINDING = "__WDL_DO_ALARMS__"; const ALARM_TABLE = "_wdl_do_alarms"; +// This module is evaluated before tenant code. Keep the small set of intrinsics +// that controls alarm classification, state transitions, and facade installation +// stable after tenant top-level evaluation mutates the shared isolate realm. +const NativeDate = Date; +const NativeNumber = Number; +const NativePromise = Promise; +const NativeProxy = Proxy; +const NativeResponse = Response; +const NativeString = String; +const nativeCrypto = crypto; +const arrayAt = Array.prototype.at; +const arrayIsArray = Array.isArray; +const arrayPush = Array.prototype.push; +const cryptoRandomUUID = crypto.randomUUID; +const dateGetTime = Date.prototype.getTime; +const dateNow = Date.now; +const headersGet = Headers.prototype.get; +const mapForEach = Map.prototype.forEach; +const mathTrunc = Math.trunc; +const numberIsFinite = Number.isFinite; +const numberIsInteger = Number.isInteger; +const objectDefineProperty = Object.defineProperty; +const promiseResolve = Promise.resolve; +const reflectApply = Reflect.apply; +const reflectGet = Reflect.get; +const requestHeadersGetter = Object.getOwnPropertyDescriptor(Request.prototype, "headers").get; +const requestJson = Request.prototype.json; +const responseJson = Response.json; +const stringReplaceAll = String.prototype.replaceAll; +const stringStartsWith = String.prototype.startsWith; +const stringToLowerCase = String.prototype.toLowerCase; + function withoutInternalEnv(env) { if (!env || typeof env !== "object" || !(ALARMS_BINDING in env)) return env; const out = { ...env }; @@ -11,57 +44,87 @@ function withoutInternalEnv(env) { } function objectNameFromCtx(ctx) { - return String(ctx.id); + return NativeString(ctx.id); } function scheduledTimeFromInput(value) { - const scheduledTime = value instanceof Date ? value.getTime() : Number(value); - if (!Number.isFinite(scheduledTime) || scheduledTime <= 0) { + let scheduledTime; + try { + scheduledTime = reflectApply(dateGetTime, value, []); + } catch { + scheduledTime = NativeNumber(value); + } + if (!numberIsFinite(scheduledTime) || scheduledTime <= 0) { throw new TypeError("setAlarm() cannot be called with an alarm time <= 0"); } return scheduledTime; } function retryCountFromInput(value) { - const retryCount = Number(value ?? 0); - if (!Number.isInteger(retryCount) || retryCount < 0) { + const retryCount = NativeNumber(value ?? 0); + if (!numberIsInteger(retryCount) || retryCount < 0) { throw new TypeError("DO alarm retryCount must be a non-negative integer"); } return retryCount; } function alarmFieldsFromRow(row) { - const scheduledTime = Number(row.scheduled_time); - const retryCount = Number(row.retry_count); - if (!Number.isFinite(scheduledTime) || scheduledTime <= 0) return null; - if (!Number.isInteger(retryCount) || retryCount < 0) return null; + const scheduledTime = NativeNumber(row.scheduled_time); + const retryCount = NativeNumber(row.retry_count); + if (!numberIsFinite(scheduledTime) || scheduledTime <= 0) return null; + if (!numberIsInteger(retryCount) || retryCount < 0) return null; return { scheduledTime, retryCount }; } function alarmToken() { - return crypto.randomUUID(); + return reflectApply(cryptoRandomUUID, nativeCrypto, []); +} + +function safeErrorField(err, field) { + try { + return err == null ? undefined : err[field]; + } catch { + return undefined; + } +} + +function safeErrorString(value) { + try { + return value == null ? null : NativeString(value); + } catch { + return null; + } } function formatWrappedError(err) { - const out = { - error_name: err?.name || "Error", - error_message: err instanceof Error ? err.message : String(err), - }; - if (err?.code != null) out.error_code = String(err.code); - return out; + try { + const out = { + error_name: safeErrorString(safeErrorField(err, "name")) || "Error", + error_message: safeErrorString(safeErrorField(err, "message")) || safeErrorString(err) || "Unknown error", + }; + const code = safeErrorString(safeErrorField(err, "code")); + if (code != null) out.error_code = code; + return out; + } catch { + return { error_name: "Error", error_message: "Unknown error" }; + } } function logStructured(level, event, fields = {}) { - const payload = { - ts: new Date().toISOString(), - service: "do-runtime", - level, - event, - ...fields, - }; - const line = JSON.stringify(payload); - if (level === "error") console.error(line); - else console.log(line); + try { + const payload = { + ts: new Date().toISOString(), + service: "do-runtime", + level, + event, + ...fields, + }; + const line = JSON.stringify(payload); + if (level === "error") console.error(line); + else console.log(line); + } catch { + // Tenant mutations must not turn best-effort logging into storage behavior. + } } function ensureAlarmTable(storage) { @@ -79,9 +142,11 @@ function ensureAlarmTable(storage) { function readAlarmRow(storage) { ensureAlarmTable(storage); - return [...storage.sql.exec( + const result = storage.sql.exec( "SELECT scheduled_time, retry_count, in_flight, token FROM " + ALARM_TABLE + " WHERE id = 1" - )][0] || null; + ); + const rows = arrayIsArray(result) ? result : [...result]; + return rows[0] || null; } function writeAlarmRow(storage, row) { @@ -95,11 +160,11 @@ function writeAlarmRow(storage, row) { "in_flight = excluded.in_flight, " + "token = excluded.token, " + "last_error = excluded.last_error", - Math.trunc(scheduledTimeFromInput(row.scheduledTime)), + mathTrunc(scheduledTimeFromInput(row.scheduledTime)), retryCountFromInput(row.retryCount), row.inFlight ? 1 : 0, - String(row.token), - row.lastError == null ? null : String(row.lastError) + NativeString(row.token), + row.lastError == null ? null : NativeString(row.lastError) ); } @@ -109,13 +174,13 @@ function deleteAlarmRow(storage, token = null) { storage.sql.exec("DELETE FROM " + ALARM_TABLE + " WHERE id = 1"); return; } - storage.sql.exec("DELETE FROM " + ALARM_TABLE + " WHERE id = 1 AND token = ?", String(token)); + storage.sql.exec("DELETE FROM " + ALARM_TABLE + " WHERE id = 1 AND token = ?", NativeString(token)); } async function flushAlarmSideEffects(sideEffects) { // One object has one alarm row. Transactional alarm updates coalesce to the // final SQLite row, so only the final backend index side effect should run. - const finalEffect = sideEffects.at(-1); + const finalEffect = reflectApply(arrayAt, sideEffects, [-1]); if (finalEffect) await finalEffect(); } @@ -136,7 +201,7 @@ async function setStorageAlarm(storage, alarmBinding, className, objectName, sch token, }); if (sideEffects) { - sideEffects.push(async () => { + reflectApply(arrayPush, sideEffects, [async () => { try { await effect(); } catch (err) { @@ -145,7 +210,7 @@ async function setStorageAlarm(storage, alarmBinding, className, objectName, sch deleteAlarmRow(storage, token); throw err; } - }); + }]); } else { try { await effect(); @@ -163,12 +228,12 @@ async function deleteStorageAlarm(storage, alarmBinding, className, objectName, deleteAlarmRow(storage); const token = sideEffects?.baselineAlarmToken != null ? sideEffects.baselineAlarmToken - : row?.token == null ? null : String(row.token); + : row?.token == null ? null : NativeString(row.token); const effect = () => token ? alarmBinding.deleteAlarmIndex({ className, objectName, token }) - : Promise.resolve("skipped"); + : reflectApply(promiseResolve, NativePromise, ["skipped"]); if (sideEffects) { - sideEffects.push(async () => { + reflectApply(arrayPush, sideEffects, [async () => { try { await effect(); } catch (err) { @@ -176,13 +241,13 @@ async function deleteStorageAlarm(storage, alarmBinding, className, objectName, writeAlarmRow(storage, { scheduledTime: row.scheduled_time, retryCount: row.retry_count, - inFlight: Number(row.in_flight) === 1, + inFlight: NativeNumber(row.in_flight) === 1, token: row.token, }); } throw err; } - }); + }]); } else { try { await effect(); @@ -191,7 +256,7 @@ async function deleteStorageAlarm(storage, alarmBinding, className, objectName, writeAlarmRow(storage, { scheduledTime: row.scheduled_time, retryCount: row.retry_count, - inFlight: Number(row.in_flight) === 1, + inFlight: NativeNumber(row.in_flight) === 1, token: row.token, }); } @@ -202,7 +267,7 @@ async function deleteStorageAlarm(storage, alarmBinding, className, objectName, async function getStorageAlarm(storage, alarmBinding, className, objectName) { const row = readAlarmRow(storage); - if (!row || Number(row.in_flight) === 1) return null; + if (!row || NativeNumber(row.in_flight) === 1) return null; const fields = alarmFieldsFromRow(row); if (!fields) { deleteAlarmRow(storage); @@ -214,7 +279,7 @@ async function getStorageAlarm(storage, alarmBinding, className, objectName) { objectName, scheduledTime: fields.scheduledTime, retryCount: fields.retryCount, - token: String(row.token), + token: NativeString(row.token), }); } catch (err) { logStructured("warn", "do_alarm_index_repair_failed", { @@ -234,11 +299,11 @@ function claimStorageAlarm(storage, alarm) { deleteAlarmRow(storage); return null; } - const rowToken = String(row.token); - const alarmTokenValue = alarm?.token == null ? null : String(alarm.token); + const rowToken = NativeString(row.token); + const alarmTokenValue = alarm?.token == null ? null : NativeString(alarm.token); if (alarmTokenValue && alarmTokenValue !== rowToken) return null; const retryCount = retryCountFromInput(alarm?.retryCount ?? fields.retryCount); - if (Number(row.in_flight) !== 1 && fields.scheduledTime > Date.now()) return null; + if (NativeNumber(row.in_flight) !== 1 && fields.scheduledTime > reflectApply(dateNow, NativeDate, [])) return null; writeAlarmRow(storage, { scheduledTime: fields.scheduledTime, retryCount, @@ -253,14 +318,17 @@ function completeStorageAlarm(storage, token) { } function quoteSqlIdentifier(name) { - return '"' + String(name).replaceAll('"', '""') + '"'; + return '"' + reflectApply(stringReplaceAll, NativeString(name), ['"', '""']) + '"'; } function sqlObjectDropStatement(row) { - const type = String(row.type); - const name = String(row.name); - const lowerName = name.toLowerCase(); - if (lowerName.startsWith("sqlite_") || lowerName.startsWith("_cf_")) return null; + const type = NativeString(row.type); + const name = NativeString(row.name); + const lowerName = reflectApply(stringToLowerCase, name, []); + if ( + reflectApply(stringStartsWith, lowerName, ["sqlite_"]) || + reflectApply(stringStartsWith, lowerName, ["_cf_"]) + ) return null; if (type === "table") return "DROP TABLE IF EXISTS " + quoteSqlIdentifier(name); if (type === "view") return "DROP VIEW IF EXISTS " + quoteSqlIdentifier(name); if (type === "trigger") return "DROP TRIGGER IF EXISTS " + quoteSqlIdentifier(name); @@ -270,22 +338,27 @@ function sqlObjectDropStatement(row) { async function deleteAllKvStorage(storage) { const entries = await storage.list(); - const keys = [...entries.keys()]; + const keys = []; + reflectApply(mapForEach, entries, [(_value, key) => { + reflectApply(arrayPush, keys, [key]); + }]); if (keys.length) await storage.delete(keys); } function deleteAllSqlStorage(storage, deleteAlarm) { - const rows = [...storage.sql.exec( + const result = storage.sql.exec( "SELECT type, name FROM sqlite_master " + "WHERE type IN ('trigger', 'view', 'table', 'index') " + "ORDER BY CASE type WHEN 'trigger' THEN 0 WHEN 'view' THEN 1 WHEN 'table' THEN 2 ELSE 3 END" - )]; + ); + const rows = arrayIsArray(result) ? result : [...result]; // workerd permits this connection-level PRAGMA; integration coverage pins it // because database-level PRAGMA writes are not part of the public DO SQL API. storage.sql.exec("PRAGMA foreign_keys = OFF"); try { - for (const row of rows) { - if (String(row.name) === ALARM_TABLE && !deleteAlarm) continue; + for (let i = 0; i < rows.length; i += 1) { + const row = rows[i]; + if (NativeString(row.name) === ALARM_TABLE && !deleteAlarm) continue; const statement = sqlObjectDropStatement(row); if (statement) storage.sql.exec(statement); } @@ -298,7 +371,7 @@ function wrapStorage(storage, alarmBinding, className, objectName, sideEffects = if (!storage || !alarmBinding) return storage; let syncTransactionSideEffects = null; const activeSideEffects = () => syncTransactionSideEffects || sideEffects; - return new Proxy(storage, { + return new NativeProxy(storage, { get(target, prop, receiver) { if (prop === "setAlarm") { return (scheduledTime, _options = undefined) => { @@ -325,11 +398,11 @@ function wrapStorage(storage, alarmBinding, className, objectName, sideEffects = return async (callback, ...rest) => { const txSideEffects = []; const baselineAlarm = readAlarmRow(alarmStorage); - txSideEffects.baselineAlarmToken = baselineAlarm?.token == null ? null : String(baselineAlarm.token); + txSideEffects.baselineAlarmToken = baselineAlarm?.token == null ? null : NativeString(baselineAlarm.token); const wrapped = typeof callback === "function" ? (txn) => callback(wrapStorage(txn, alarmBinding, className, objectName, txSideEffects, target)) : callback; - const result = await Reflect.apply(Reflect.get(target, prop, receiver), target, [wrapped, ...rest]); + const result = await reflectApply(reflectGet(target, prop, receiver), target, [wrapped, ...rest]); await flushAlarmSideEffects(txSideEffects); return result; }; @@ -347,7 +420,7 @@ function wrapStorage(storage, alarmBinding, className, objectName, sideEffects = syncTransactionSideEffects = previousSideEffects; } } : callback; - const result = Reflect.apply(Reflect.get(target, prop, receiver), target, [wrapped, ...rest]); + const result = reflectApply(reflectGet(target, prop, receiver), target, [wrapped, ...rest]); return result; }; } @@ -357,8 +430,8 @@ function wrapStorage(storage, alarmBinding, className, objectName, sideEffects = throw new TypeError("deleteAll() cannot be used inside transactionSync(); use transaction()"); } return (async () => { - const [options, ...rest] = args; - if (rest.length) throw new TypeError("deleteAll() accepts at most one options argument"); + if (args.length > 1) throw new TypeError("deleteAll() accepts at most one options argument"); + const options = args[0]; const deleteAlarm = options?.deleteAlarm !== false; const alarmRow = readAlarmRow(alarmStorage); const preservedAlarm = deleteAlarm ? null : alarmRow; @@ -370,25 +443,25 @@ function wrapStorage(storage, alarmBinding, className, objectName, sideEffects = const sideEffects = activeSideEffects(); const token = sideEffects?.baselineAlarmToken != null ? sideEffects.baselineAlarmToken - : alarmRow?.token == null ? null : String(alarmRow.token); + : alarmRow?.token == null ? null : NativeString(alarmRow.token); const effect = () => token ? alarmBinding.deleteAlarmIndex({ className, objectName, token }) - : Promise.resolve("skipped"); - if (sideEffects) sideEffects.push(effect); + : reflectApply(promiseResolve, NativePromise, ["skipped"]); + if (sideEffects) reflectApply(arrayPush, sideEffects, [effect]); else await effect(); } else if (preservedAlarm) { writeAlarmRow(alarmStorage, { scheduledTime: preservedAlarm.scheduled_time, retryCount: preservedAlarm.retry_count, - inFlight: Number(preservedAlarm.in_flight) === 1, + inFlight: NativeNumber(preservedAlarm.in_flight) === 1, token: preservedAlarm.token, }); } })(); }; } - const value = Reflect.get(target, prop, receiver); - return typeof value === "function" ? value.bind(target) : value; + const value = reflectGet(target, prop, receiver); + return typeof value === "function" ? (...args) => reflectApply(value, target, args) : value; }, }); } @@ -397,15 +470,15 @@ function wrapCtx(ctx, alarmBinding, className) { if (!ctx || !alarmBinding) return ctx; const objectName = objectNameFromCtx(ctx); let storageProxy = null; - return new Proxy(ctx, { + return new NativeProxy(ctx, { get(target, prop, receiver) { if (prop === "storage") { - const storage = Reflect.get(target, prop, receiver); + const storage = reflectGet(target, prop, receiver); if (!storageProxy) storageProxy = wrapStorage(storage, alarmBinding, className, objectName); return storageProxy; } - const value = Reflect.get(target, prop, receiver); - return typeof value === "function" ? value.bind(target) : value; + const value = reflectGet(target, prop, receiver); + return typeof value === "function" ? (...args) => reflectApply(value, target, args) : value; }, }); } @@ -415,7 +488,7 @@ function installStorageProxy(ctx, alarmBinding, className) { const objectName = objectNameFromCtx(ctx); const storageProxy = wrapStorage(ctx.storage, alarmBinding, className, objectName); try { - Object.defineProperty(ctx, "storage", { + objectDefineProperty(ctx, "storage", { value: storageProxy, configurable: true, }); @@ -440,9 +513,12 @@ export function wrapDurableObjectClass(Base, className) { // only the alarm binding here so __WDL_HOST_BINDINGS_WRAPPED can survive // through the two-layer wrapper contract. super(constructorCtx, withoutInternalEnv(env)); + // Resolve once through the host wrapper after construction so prototype + // methods, class fields, and accessors retain the real instance receiver. + const tenantFetch = reflectGet(this, "fetch", this); const wrappedCtx = wrapCtx(constructorCtx, alarmBinding, className); try { - Object.defineProperty(this, "ctx", { + objectDefineProperty(this, "ctx", { value: wrappedCtx, configurable: true, writable: true, @@ -450,26 +526,57 @@ export function wrapDurableObjectClass(Base, className) { } catch { this.ctx = wrappedCtx; } - } - - async fetch(request) { - if (request.headers.get(ALARM_HEADER) === "1") { - const alarm = await request.json(); - const claim = claimStorageAlarm(this.ctx.storage, alarm); - if (!claim) return Response.json({ ok: true, ignored: true }); - if (typeof super.alarm === "function") { - await super.alarm({ - retryCount: claim.retryCount, - isRetry: claim.retryCount > 0, - }); - } - completeStorageAlarm(this.ctx.storage, claim.token); - return Response.json({ ok: true }); - } - if (typeof super.fetch !== "function") { - return new Response("Durable Object class has no fetch handler", { status: 500 }); - } - return await super.fetch(request); + // workerd wraps instance handlers after construction, so this platform + // dispatch must remain writable and configurable after replacing class fields. + objectDefineProperty(this, "fetch", { + value: async function(request) { + const headers = reflectApply(requestHeadersGetter, request, []); + if (reflectApply(headersGet, headers, [ALARM_HEADER]) === "1") { + const alarm = await reflectApply(requestJson, request, []); + const claim = claimStorageAlarm(this.ctx.storage, alarm); + if (!claim) return reflectApply(responseJson, NativeResponse, [{ ok: true, ignored: true }]); + // Alarm accessors may depend on initialized instance state. + const tenantAlarm = reflectGet(this, "alarm", this); + if (typeof tenantAlarm === "function") { + await reflectApply(tenantAlarm, this, [{ + retryCount: claim.retryCount, + isRetry: claim.retryCount > 0, + }]); + } + completeStorageAlarm(this.ctx.storage, claim.token); + return reflectApply(responseJson, NativeResponse, [{ ok: true }]); + } + if (reflectApply(headersGet, headers, [RPC_HEADER]) === "1") { + const rpc = await reflectApply(requestJson, request, []); + try { + const tenantMethod = reflectGet(this, rpc.method, this); + if (typeof tenantMethod !== "function") { + return reflectApply(responseJson, NativeResponse, [{ + error: "do_rpc_method_not_found", + message: "Durable Object RPC method " + rpc.method + " was not found", + }, { status: 404 }]); + } + const result = await reflectApply(tenantMethod, this, rpc.args); + return reflectApply(responseJson, NativeResponse, [{ ok: true, result }]); + } catch (err) { + const formatted = formatWrappedError(err); + const stack = safeErrorString(safeErrorField(err, "stack")); + return reflectApply(responseJson, NativeResponse, [{ + error: "do_rpc_error", + name: formatted.error_name, + message: formatted.error_message, + ...(stack ? { stack } : {}), + }, { status: 500 }]); + } + } + if (typeof tenantFetch !== "function") { + return new NativeResponse("Durable Object class has no fetch handler", { status: 500 }); + } + return await reflectApply(tenantFetch, this, [request]); + }, + configurable: true, + writable: true, + }); } }; } diff --git a/do-runtime/alarm.js b/do-runtime/alarm.js index ee77e6c..4672a65 100644 --- a/do-runtime/alarm.js +++ b/do-runtime/alarm.js @@ -1,4 +1,8 @@ -import { DoRuntimeError, nonEmptyAlarmString } from "do-runtime-protocol"; +import { + DoRuntimeError, + isWellFormedUnicodeString, + nonEmptyAlarmString, +} from "do-runtime-protocol"; import { errorMessage } from "shared-errors"; import { withInternalAuth } from "shared-internal-auth"; @@ -38,6 +42,9 @@ function requiredString(value, field) { if (text === null) { throw new TypeError(`DO alarm ${field} must be a non-empty string`); } + if (!isWellFormedUnicodeString(text)) { + throw new TypeError(`DO alarm ${field} must contain well-formed Unicode`); + } return text; } diff --git a/do-runtime/config.capnp b/do-runtime/config.capnp index c91cac1..50d5f82 100644 --- a/do-runtime/config.capnp +++ b/do-runtime/config.capnp @@ -63,10 +63,11 @@ const doRuntimeWorker :Workerd.Worker = ( (name = "runtime-bindings-do", esModule = embed "../runtime/bindings/do.js"), (name = "runtime-bindings-internal-auth-backend", esModule = embed "../runtime/bindings/internal-auth-backend.js"), (name = "runtime-do-transport", esModule = embed "../runtime/_wdl-do-transport.js"), - (name = "runtime-owner-endpoint", esModule = embed "../runtime/_wdl-owner-endpoint.js"), - # _wdl-do-transport.js imports the same helper by relative file path, while - # host bindings import it by bare module name. Workerd needs both entries. - (name = "_wdl-owner-endpoint.js", esModule = embed "../runtime/_wdl-owner-endpoint.js"), + (name = "_wdl-request-id.js", esModule = embed "../runtime/_wdl-request-id.js"), + (name = "shared-owner-endpoint", esModule = embed "../shared/owner-endpoint.js"), + # Injected DO transport uses a relative module name; host modules use the + # shared bare name. Both resolve to the same shared contract owner. + (name = "_wdl-owner-endpoint.js", esModule = embed "../shared/owner-endpoint.js"), (name = "runtime-owner-hint-cache", esModule = embed "../runtime/_wdl-owner-hint-cache.js"), (name = "runtime-metrics", esModule = embed "../runtime/metrics.js"), (name = "runtime-state", esModule = embed "../runtime/runtime.js"), @@ -80,7 +81,7 @@ const doRuntimeWorker :Workerd.Worker = ( (name = "runtime-r2-utils-source", text = embed "../runtime/r2-utils.js"), (name = "runtime-do-client-source", text = embed "../runtime/do-client.js"), (name = "runtime-do-transport-source", text = embed "../runtime/_wdl-do-transport.js"), - (name = "runtime-owner-endpoint-source", text = embed "../runtime/_wdl-owner-endpoint.js"), + (name = "runtime-owner-endpoint-source", text = embed "../shared/owner-endpoint.js"), (name = "runtime-owner-hint-cache-source", text = embed "../runtime/_wdl-owner-hint-cache.js"), (name = "runtime-request-id-source", text = embed "../runtime/_wdl-request-id.js"), (name = "runtime-workflows-client-source", text = embed "../runtime/workflows-client.js"), @@ -98,6 +99,7 @@ const doRuntimeWorker :Workerd.Worker = ( (name = "shared-s3-xml", esModule = embed "../shared/s3-xml.js"), (name = "shared-owner-forwarder", esModule = embed "../shared/owner-forwarder.js"), (name = "shared-owner-lease", esModule = embed "../shared/owner-lease.js"), + (name = "shared-optimistic-retry", esModule = embed "../shared/optimistic-retry.js"), (name = "shared-owner-protocol", esModule = embed "../shared/owner-protocol.js"), (name = "shared-redis", esModule = embed "../shared/redis.js"), (name = "shared-redis-command-client", esModule = embed "../shared/redis-command-client.js"), @@ -108,7 +110,10 @@ const doRuntimeWorker :Workerd.Worker = ( (name = "hex.js", esModule = embed "../shared/hex.js"), (name = "errors.js", esModule = embed "../shared/errors.js"), (name = "shared-errors", esModule = embed "../shared/errors.js"), + (name = "shared-base64", esModule = embed "../shared/base64.js"), (name = "shared-respond", esModule = embed "../shared/respond.js"), + (name = "shared-s3-retry", esModule = embed "../shared/s3-retry.js"), + (name = "respond.js", esModule = embed "../shared/respond.js"), (name = "shared-bounded-body", esModule = embed "../shared/bounded-body.js"), (name = "shared-request-scope", esModule = embed "../shared/request-scope.js"), (name = "shared-task-identity", esModule = embed "../shared/task-identity.js"), @@ -149,7 +154,6 @@ const doRuntimeWorker :Workerd.Worker = ( (name = "R2_S3_REGION", fromEnvironment = "R2_S3_REGION"), (name = "DO_TASK_ID", fromEnvironment = "DO_TASK_ID"), (name = "DO_TASK_ENDPOINT", fromEnvironment = "DO_TASK_ENDPOINT"), - (name = "DO_TASK_PORT", fromEnvironment = "DO_TASK_PORT"), (name = "DO_TASK_IDENTITY_TIMEOUT_MS", fromEnvironment = "DO_TASK_IDENTITY_TIMEOUT_MS"), (name = "DO_TASK_CONTAINER_NAME", fromEnvironment = "DO_TASK_CONTAINER_NAME"), (name = "ECS_CONTAINER_METADATA_URI_V4", fromEnvironment = "ECS_CONTAINER_METADATA_URI_V4"), @@ -157,7 +161,6 @@ const doRuntimeWorker :Workerd.Worker = ( (name = "DO_OWNER_LEASE_GUARD_MS", fromEnvironment = "DO_OWNER_LEASE_GUARD_MS"), (name = "DO_RENEW_CONCURRENCY", fromEnvironment = "DO_RENEW_CONCURRENCY"), (name = "DO_DRAIN_IN_FLIGHT_TIMEOUT_MS", fromEnvironment = "DO_DRAIN_IN_FLIGHT_TIMEOUT_MS"), - (name = "DO_TEST_HOOKS", fromEnvironment = "DO_TEST_HOOKS"), (name = "LOG_LEVEL", fromEnvironment = "LOG_LEVEL"), ], ); diff --git a/do-runtime/index.js b/do-runtime/index.js index b5be45a..4828836 100644 --- a/do-runtime/index.js +++ b/do-runtime/index.js @@ -4,14 +4,15 @@ import { verifyInternalAuthHeaders, } from "shared-internal-auth"; import { createHttpRequestScope } from "shared-request-scope"; -import { discardResponseBody, prometheusResponse } from "shared-respond"; +import { discardResponseBody, prometheusResponse, rebuildResponseWithHeaders } from "shared-respond"; import { boundedPositiveIntEnv } from "shared-owner-lease"; +import { parseVersion } from "shared-version"; import { formatWorkerId } from "shared-worker-id"; import { WdlDoHostActor } from "do-runtime-actor"; import { handleAlarmDispatch } from "do-runtime-alarm-dispatch"; import { buildLocalActorRequest, - doErrorResponse, + doPlatformErrorResponse, DoRuntimeError, hostIdForObject, hostIdForShard, @@ -72,7 +73,7 @@ const STORAGE_DELETE_REQUEST = { }; /** - * @typedef {Record & { LOG_LEVEL?: unknown, REDIS_ADDR?: string, DO_HOSTS: DurableObjectNamespace, DO_TEST_HOOKS?: unknown, DO_DRAIN_IN_FLIGHT_TIMEOUT_MS?: unknown, DO_RENEW_INTERVAL_MS?: unknown }} DoEnv + * @typedef {Record & { LOG_LEVEL?: unknown, REDIS_ADDR?: string, DO_HOSTS: DurableObjectNamespace, DO_DRAIN_IN_FLIGHT_TIMEOUT_MS?: unknown, DO_RENEW_INTERVAL_MS?: unknown }} DoEnv * @typedef {import("do-runtime-protocol").DoInvoke} DoInvoke * @typedef {{ ownerKey: string, hostId?: string, className?: string, ns: string, worker: string, doStorageId: string, taskId: string, endpoint: string, generation: number, leaseExpiresAt?: number }} DoOwner * @typedef {{ requestId?: string | null, hopCount?: number, forwardPath?: string, localUrl?: string, request?: { method: string, url: string, headers: Array<[string, string]> } | null, metricKind?: string | null, acceptOwnerHint?: boolean }} DispatchOptions @@ -153,14 +154,7 @@ function withOwnerHintHeaders(response, owner) { for (const [name, value] of Object.entries(ownerHintHeaders(owner))) { headers.set(name, value); } - const init = /** @type {ResponseInit & { webSocket?: WebSocket }} */ ({ - status: response.status, - statusText: response.statusText, - headers, - }); - const webSocket = /** @type {{ webSocket?: WebSocket }} */ (response).webSocket; - if (webSocket) init.webSocket = webSocket; - return new Response(response.status === 101 ? null : response.body, init); + return rebuildResponseWithHeaders(response, headers); } /** @param {Request} request */ @@ -245,9 +239,7 @@ async function dispatchStorageDelete(env, invoke, requestId = null, hopCount = 0 * @param {string} requestId */ async function handleInvoke(request, env, requestId) { - const invoke = await readDoInvokeRequest(request, { - allowInlineWorkerCode: env.DO_TEST_HOOKS === "1", - }); + const invoke = await readDoInvokeRequest(request); return await dispatchInvoke( env, invoke, @@ -364,9 +356,7 @@ async function handleDrain(env) { * @param {string} requestId */ async function handleStorageDelete(request, env, requestId) { - const invoke = await readDoInvokeRequest(request, { - allowInlineWorkerCode: false, - }); + const invoke = await readDoInvokeRequest(request); return await dispatchStorageDelete( env, invoke, @@ -430,6 +420,9 @@ async function handleStorageDeleteWorker(env, request, requestId = null) { if (!ns || !worker || !version || !doStorageId) { return jsonError(400, "invalid_request", "ns, worker, version, and doStorageId are required"); } + if (parseVersion(version) == null) { + return jsonError(400, "invalid_request", "version is invalid"); + } let deleted = 0; /** @type {Array>} */ @@ -543,7 +536,7 @@ export default { return scope.respond(jsonError(404, "not_found", "Not found")); } catch (err) { scope.markError(err); - return scope.respond(doErrorResponse(err)); + return scope.respond(doPlatformErrorResponse(err)); } finally { scope.complete(); } diff --git a/do-runtime/load-code-budget.js b/do-runtime/load-code-budget.js index 7e989c0..ff272bc 100644 --- a/do-runtime/load-code-budget.js +++ b/do-runtime/load-code-budget.js @@ -31,14 +31,20 @@ export function hasDoRuntimeInjectedModules(meta) { function generateDoRuntimeWrapperModule(userMainSpecifier, classNames) { const userMain = JSON.stringify(`./${userMainSpecifier}`); const alarmShim = JSON.stringify(`./${DO_ALARM_SHIM_MODULE}`); - const wrappedClasses = classNames.map((name) => ` -export class ${name} extends wrapDurableObjectClass(user.${name}, ${JSON.stringify(name)}) {} + const wrappedClasses = classNames.map((name, index) => ` +const __WdlWrappedDurableObject${index}__ = ({ + [${JSON.stringify(name)}]: class extends __WdlWrapDurableObjectClass__( + __WdlUserModule__.${name}, + ${JSON.stringify(name)} + ) {}, +})[${JSON.stringify(name)}]; +export { __WdlWrappedDurableObject${index}__ as ${name} }; `).join(""); return ` -import * as user from ${userMain}; -export * from ${userMain}; +import { wrapDurableObjectClass as __WdlWrapDurableObjectClass__ } from ${alarmShim}; -import { wrapDurableObjectClass } from ${alarmShim}; +import * as __WdlUserModule__ from ${userMain}; +export * from ${userMain}; ${wrappedClasses} `; diff --git a/do-runtime/owner-client.js b/do-runtime/owner-client.js index 9dba006..70ea4e1 100644 --- a/do-runtime/owner-client.js +++ b/do-runtime/owner-client.js @@ -1,5 +1,6 @@ import { DO_INVOKE_CONTENT_TYPE, + DO_OWNERSHIP_CODE, DoRuntimeError, encodeDoInvokeRequest, } from "do-runtime-protocol"; @@ -23,6 +24,8 @@ export function parseHopCount(value) { return parseForwardHopCount(value); } +const DO_OWNER_PORT = 8788; + /** @param {DoOwner} owner */ function ownerFence(owner) { return { @@ -44,6 +47,8 @@ export async function forwardToOwner(invoke, env, owner, requestId = null, hopCo return await forwardOwnerRequest({ env, endpoint: owner.endpoint, + endpointPort: DO_OWNER_PORT, + endpointService: "do-runtime", pathname, requestId, hopCount, @@ -68,14 +73,16 @@ export async function forwardToOwner(invoke, env, owner, requestId = null, hopCo path: pathname, }), missingEndpointError: () => - new DoRuntimeError(503, "owner_endpoint_missing", `DO scope ${owner.ownerKey} owner has no endpoint`), + new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_ENDPOINT_MISSING, `DO scope ${owner.ownerKey} owner has no endpoint`), + invalidEndpointError: () => + new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_UNAVAILABLE, `DO scope ${owner.ownerKey} owner endpoint is invalid`), hopExhaustedError: () => new DoRuntimeError( 503, - "forward_hop_exhausted", + DO_OWNERSHIP_CODE.FORWARD_HOP_EXHAUSTED, `DO scope ${owner.ownerKey} exceeded the maximum forward depth for ${pathname}` ), - unavailableError: () => new DoRuntimeError(503, "owner_unavailable", "DO owner is unavailable"), + unavailableError: () => new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_UNAVAILABLE, "DO owner is unavailable"), }); } @@ -91,6 +98,8 @@ export async function forwardConnectToOwner(request, invoke, env, owner, request return await forwardOwnerRequest({ env, endpoint: owner.endpoint, + endpointPort: DO_OWNER_PORT, + endpointService: "do-runtime", pathname: "/internal/do/connect", method: request.method, requestId, @@ -118,13 +127,15 @@ export async function forwardConnectToOwner(request, invoke, env, owner, request owner_endpoint: owner.endpoint, }), missingEndpointError: () => - new DoRuntimeError(503, "owner_endpoint_missing", `DO scope ${owner.ownerKey} owner has no endpoint`), + new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_ENDPOINT_MISSING, `DO scope ${owner.ownerKey} owner has no endpoint`), + invalidEndpointError: () => + new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_UNAVAILABLE, `DO scope ${owner.ownerKey} owner endpoint is invalid`), hopExhaustedError: () => new DoRuntimeError( 503, - "forward_hop_exhausted", + DO_OWNERSHIP_CODE.FORWARD_HOP_EXHAUSTED, `DO scope ${owner.ownerKey} exceeded the maximum forward depth` ), - unavailableError: () => new DoRuntimeError(503, "owner_unavailable", "DO owner is unavailable"), + unavailableError: () => new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_UNAVAILABLE, "DO owner is unavailable"), }); } diff --git a/do-runtime/owner-registry.js b/do-runtime/owner-registry.js index 64fbd5c..c87d3b2 100644 --- a/do-runtime/owner-registry.js +++ b/do-runtime/owner-registry.js @@ -4,7 +4,9 @@ import { } from "shared-redis"; import { buildOwnerKey, + DO_OWNERSHIP_CODE, DoRuntimeError, + hostIdForShard, } from "do-runtime-protocol"; import { resolveTaskIdentity, @@ -20,6 +22,8 @@ import { import { createRedisClient } from "do-runtime-redis"; import { envValueOr } from "shared-env"; import { errorMessage } from "shared-errors"; +import { isValidRuntimeLoadNs, WORKER_NAME_RE } from "shared-ns-pattern"; +import { validOwnerEndpointForService } from "shared-owner-endpoint"; import { boundedPositiveIntEnv, currentOwnerGenerationCounter, @@ -34,25 +38,33 @@ import { ownerProtocolKeys, readOwnerRecord, readOwnerRecordWithRedisTime, + readOwnerSnapshotWithRedisTime, stageOwnerClaim, stageOwnerRelease, stageOwnerRenew, } from "shared-owner-protocol"; -import { doStorageIdKey } from "shared-version"; +import { + DO_OWNER_SCOPE_PREFIX, + VERSION_DELETE_LOCK_KIND, + deleteLockKey, + doStorageIdKey, + parseDeleteLockKind, +} from "shared-version"; const DEFAULT_OWNER_TTL_SECONDS = 120; const DEFAULT_RENEW_CONCURRENCY = 8; const MAX_RENEW_CONCURRENCY = 64; const DEFAULT_OWNER_LEASE_GUARD_MS = 1_000; -const OWNER_PREFIX = "do:owner:scope:"; const OWNER_CLAIM_RETRIES = 3; const OWNER_RENEW_FRACTION = 0.5; +const DO_OWNER_PORT = 8788; /** * @typedef {Record & { REDIS_ADDR?: unknown, REDIS_DB?: unknown, DO_OWNER_TTL_SECONDS?: unknown, DO_OWNER_LEASE_GUARD_MS?: unknown, DO_RENEW_CONCURRENCY?: unknown }} DoEnv * @typedef {{ taskId: string, endpoint: string }} LocalTask * @typedef {{ ownerKey: string, hostId?: string, className?: string, ns: string, worker: string, doStorageId: string, taskId: string, endpoint: string, generation: number, leaseExpiresAt?: number }} DoOwner * @typedef {{ ownerKey: string, taskId: string, generation: number }} OwnerFence + * @typedef {{ ns: string, worker: string }} BundleScope * @typedef {import("do-runtime-protocol").DoInvoke} DoInvoke * @typedef {{ hostId: string, className?: string, ns: string, worker: string, doStorageId: string }} InvokeScope * @typedef {{ get(key: string): Promise, getWithTime(key: string): Promise<{ value: string | Uint8Array | null | undefined, nowMs: number }>, time(): Promise, watch?(...keys: string[]): Promise, unwatch?(): Promise, multi?(): import("shared-redis").RedisMulti }} RedisLike @@ -82,24 +94,31 @@ export function renewConcurrency(env) { /** @param {string} ownerKey */ export function ownerKeyOf(ownerKey) { - return ownerProtocolKeys(OWNER_PREFIX, ownerKey).ownerKey; + return ownerProtocolKeys(DO_OWNER_SCOPE_PREFIX, ownerKey).ownerKey; } /** @param {string} ownerKey */ export function ownerGenerationKeyOf(ownerKey) { - return ownerProtocolKeys(OWNER_PREFIX, ownerKey).generationKey; + return ownerProtocolKeys(DO_OWNER_SCOPE_PREFIX, ownerKey).generationKey; +} + +/** @param {unknown} value @returns {value is BundleScope & Record} */ +function isValidBundleScope(value) { + const scope = /** @type {Record} */ (value); + return isValidRuntimeLoadNs(scope?.ns) && + typeof scope?.worker === "string" && + WORKER_NAME_RE.test(scope.worker); } -/** @param {DoInvoke} invoke */ +/** @param {DoInvoke} invoke @returns {InvokeScope} */ function requireInvokeScope(invoke) { const record = /** @type {Record} */ (invoke); if ( typeof record.hostId !== "string" || !record.hostId || - typeof record.ns !== "string" || !record.ns || - typeof record.worker !== "string" || !record.worker || + !isValidBundleScope(record) || typeof record.doStorageId !== "string" || !record.doStorageId ) { - throw new DoRuntimeError(400, "invalid_request", "DO owner request is missing bundle scope"); + throw new DoRuntimeError(400, "invalid_request", "DO owner request has missing or invalid bundle scope"); } return { hostId: record.hostId, @@ -110,11 +129,57 @@ function requireInvokeScope(invoke) { }; } -/** @param {unknown} raw @returns {DoOwner | null} */ -export function parseOwner(raw) { - return /** @type {DoOwner | null} */ ( +/** + * @param {DoOwner | null} owner + * @param {string} expectedOwnerKey + * @param {BundleScope | null | undefined} expectedBundleScope + */ +function ownerMatchesScope(owner, expectedOwnerKey, expectedBundleScope) { + if ( + !owner || + typeof owner.ownerKey !== "string" || owner.ownerKey !== expectedOwnerKey || + typeof owner.hostId !== "string" || owner.hostId !== expectedOwnerKey || + typeof owner.className !== "string" || !owner.className || + !isValidBundleScope(owner) || + typeof owner.doStorageId !== "string" || !owner.doStorageId + ) return false; + if ( + expectedBundleScope && + (!isValidBundleScope(expectedBundleScope) || + owner.ns !== expectedBundleScope.ns || + owner.worker !== expectedBundleScope.worker) + ) return false; + const shardMarker = ":shard"; + const markerIndex = expectedOwnerKey.lastIndexOf(shardMarker); + if (markerIndex < 0) return false; + const shard = Number(expectedOwnerKey.slice(markerIndex + shardMarker.length)); + try { + return hostIdForShard(owner.doStorageId, owner.className, shard) === expectedOwnerKey; + } catch { + return false; + } +} + +/** + * @param {unknown} raw + * @param {string} expectedOwnerKey + * @param {BundleScope | null} [expectedBundleScope] + * @returns {DoOwner | null} + */ +export function parseOwner(raw, expectedOwnerKey, expectedBundleScope = null) { + if (raw == null) return null; + const owner = /** @type {DoOwner | null} */ ( parseOwnerRecord(/** @type {string | BufferSource | null | undefined} */ (raw)) ); + if ( + !owner || + !ownerMatchesScope(owner, expectedOwnerKey, expectedBundleScope) || + typeof owner.taskId !== "string" || !owner.taskId || + !validOwnerEndpointForService(owner.endpoint, DO_OWNER_PORT, "do-runtime") + ) { + throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_UNAVAILABLE, "DO owner record is invalid"); + } + return owner; } /** @@ -126,6 +191,9 @@ export function parseOwner(raw) { * @returns {DoOwner} */ function ownerRecordFor(env, invoke, localTask, generation, nowMs) { + if (!validOwnerEndpointForService(localTask.endpoint, DO_OWNER_PORT, "do-runtime")) { + throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_UNAVAILABLE, "DO owner endpoint is invalid"); + } const ttl = ownerTtlSeconds(env); const scope = requireInvokeScope(invoke); return { @@ -150,6 +218,9 @@ function ownerRecordFor(env, invoke, localTask, generation, nowMs) { * @returns {DoOwner} */ function renewedOwnerRecordFor(env, current, localTask, nowMs) { + if (!validOwnerEndpointForService(localTask.endpoint, DO_OWNER_PORT, "do-runtime")) { + throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_UNAVAILABLE, "DO owner endpoint is invalid"); + } const ttl = ownerTtlSeconds(env); return { ...current, @@ -165,7 +236,7 @@ function renewedOwnerRecordFor(env, current, localTask, nowMs) { * @returns {Promise} */ async function readOwnerFromClient(client, ownerKey) { - return await readOwnerRecord(client, ownerKeyOf(ownerKey), parseOwner); + return await readOwnerRecord(client, ownerKeyOf(ownerKey), (raw) => parseOwner(raw, ownerKey)); } /** @@ -174,15 +245,19 @@ async function readOwnerFromClient(client, ownerKey) { * @returns {Promise<{ owner: DoOwner | null, nowMs: number }>} */ async function readOwnerWithTimeFromClient(client, ownerKey) { - return await readOwnerRecordWithRedisTime(client, ownerKeyOf(ownerKey), parseOwner); + return await readOwnerRecordWithRedisTime( + client, + ownerKeyOf(ownerKey), + (raw) => parseOwner(raw, ownerKey) + ); } /** * @param {RedisLike} session - * @param {DoOwner | null | undefined} owner + * @param {Pick | null | undefined} owner */ async function ownerStoragePointerCurrent(session, owner) { - if (!owner?.ns || !owner.worker || !owner.doStorageId) return false; + if (!owner?.doStorageId || !isValidBundleScope(owner)) return false; const current = decodeBulk(await session.get(doStorageIdKey(owner.ns, owner.worker))); return current === owner.doStorageId; } @@ -274,16 +349,36 @@ export async function resolveDoOwner(env, invoke) { const ownerKey = buildOwnerKey(scope); const key = ownerKeyOf(ownerKey); const generationKey = ownerGenerationKeyOf(ownerKey); + const workerDeleteLockKey = deleteLockKey(scope.ns, scope.worker); + const storagePointerKey = doStorageIdKey(scope.ns, scope.worker); return await withWatchRetries(async () => client.session(async (session) => { - await session.watch(key, generationKey); - const { owner: current, nowMs } = await readOwnerWithTimeFromClient(session, ownerKey); + await session.watch(key, generationKey, workerDeleteLockKey); + const { owner: current, relatedValues, nowMs } = await readOwnerSnapshotWithRedisTime( + session, + key, + [workerDeleteLockKey], + (raw) => parseOwner(raw, ownerKey, scope) + ); + const deleteLockToken = decodeBulk(relatedValues[0]); + if ( + deleteLockToken != null && + parseDeleteLockKind(deleteLockToken) !== VERSION_DELETE_LOCK_KIND + ) { + await session.unwatch(); + forgetOwnedScope(ownerKey); + throw new DoRuntimeError( + 503, + DO_OWNERSHIP_CODE.STALE_OWNER_STORAGE, + `DO scope ${ownerKey} worker is being deleted` + ); + } if (current && !ownerLeaseExpired(current, nowMs)) { - await session.watch(doStorageIdKey(scope.ns, scope.worker)); + await session.watch(storagePointerKey); if (!await ownerStoragePointerCurrent(session, current)) { await session.unwatch(); forgetOwnedScope(ownerKey); - throw new DoRuntimeError(503, "stale_owner_storage", `DO scope ${ownerKey} no longer matches active worker storage`); + throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.STALE_OWNER_STORAGE, `DO scope ${ownerKey} no longer matches active worker storage`); } if (current.taskId !== localTask.taskId) { await session.unwatch(); @@ -292,7 +387,7 @@ export async function resolveDoOwner(env, invoke) { return current; } if (isDraining()) { - throw new DoRuntimeError(503, "task_draining", "DO task is draining"); + throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.TASK_DRAINING, "DO task is draining"); } const alreadyLocal = ownedScopes.has(ownerKey); const counter = alreadyLocal ? current.generation : await currentOwnerGenerationCounter(session, generationKey); @@ -315,7 +410,17 @@ export async function resolveDoOwner(env, invoke) { } if (isDraining()) { - throw new DoRuntimeError(503, "task_draining", "DO task is draining"); + throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.TASK_DRAINING, "DO task is draining"); + } + await session.watch(storagePointerKey); + if (!await ownerStoragePointerCurrent(session, scope)) { + await session.unwatch(); + forgetOwnedScope(ownerKey); + throw new DoRuntimeError( + 503, + DO_OWNERSHIP_CODE.STALE_OWNER_STORAGE, + `DO scope ${ownerKey} no longer matches active worker storage` + ); } const generation = await nextOwnerGeneration(session, generationKey, current?.generation); const owner = ownerRecordFor(env, invoke, localTask, generation, nowMs); @@ -331,7 +436,7 @@ export async function resolveDoOwner(env, invoke) { generation, }); return owner; - }), "owner_claim_raced", `failed to resolve DO owner for ${ownerKey}`); + }), DO_OWNERSHIP_CODE.OWNER_CLAIM_RACED, `failed to resolve DO owner for ${ownerKey}`); } /** @@ -342,21 +447,21 @@ export async function resolveDoOwner(env, invoke) { */ export async function assertCurrentOwnerWithLeaseBudget(env, owner, options = {}) { if (!owner?.ownerKey || !owner.taskId || owner.generation == null) { - throw new DoRuntimeError(503, "owner_fence_missing", "DO host request is missing an owner generation fence"); + throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_FENCE_MISSING, "DO host request is missing an owner generation fence"); } const client = redisClient(env); const { owner: current, nowMs } = await readOwnerWithTimeFromClient(client, owner.ownerKey); if (!current || !ownerFenceMatches(current, owner)) { forgetOwnedScope(owner.ownerKey); - throw new DoRuntimeError(503, "stale_owner_generation", `DO scope ${owner.ownerKey} owner generation is stale`); + throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.STALE_OWNER_GENERATION, `DO scope ${owner.ownerKey} owner generation is stale`); } if (ownerLeaseExpired(current, nowMs)) { forgetOwnedScope(owner.ownerKey); - throw new DoRuntimeError(503, "owner_lease_expired", `DO scope ${owner.ownerKey} owner lease has expired`); + throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_LEASE_EXPIRED, `DO scope ${owner.ownerKey} owner lease has expired`); } if (!await ownerStoragePointerCurrent(client, current)) { forgetOwnedScope(owner.ownerKey); - throw new DoRuntimeError(503, "stale_owner_storage", `DO scope ${owner.ownerKey} no longer matches active worker storage`); + throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.STALE_OWNER_STORAGE, `DO scope ${owner.ownerKey} no longer matches active worker storage`); } const leaseRemainingMs = Number(current.leaseExpiresAt ?? 0) - nowMs; if (leaseRemainingMs < ownerLeaseGuardMs(env)) { @@ -376,7 +481,7 @@ export async function assertCurrentOwnerWithLeaseBudget(env, owner, options = {} } } forgetOwnedScope(owner.ownerKey); - throw new DoRuntimeError(503, "owner_lease_too_short", `DO scope ${owner.ownerKey} owner lease has insufficient remaining budget`); + throw new DoRuntimeError(503, DO_OWNERSHIP_CODE.OWNER_LEASE_TOO_SHORT, `DO scope ${owner.ownerKey} owner lease has insufficient remaining budget`); } return { owner: current, @@ -402,9 +507,9 @@ export async function renewOwner(env, owner) { const client = redisClient(env); const key = ownerKeyOf(owner.ownerKey); return await withWatchRetries(async () => client.session(async (session) => { - // Redis WATCH observes explicit writes/deletes, not passive TTL expiry. - // Renewal is therefore a freshness optimization; generation fencing remains - // the authority if a peer claims the scope in the tiny expiry window. + // Valkey expiration deletes invalidate WATCH just like explicit writes. + // Generation fencing remains authoritative if a peer claims the scope after + // the lease expires and before this renewal commits. await session.watch(key); const { owner: current, nowMs } = await readOwnerWithTimeFromClient(session, owner.ownerKey); if (!current || !ownerFenceMatches(current, owner)) { @@ -432,7 +537,7 @@ export async function renewOwner(env, owner) { await stageOwnerRenew(session.multi(), key, renewed, ownerTtlSeconds(env)).exec(); rememberOwner(renewed); return { renewed: true, owner: renewed, nowMs }; - }), "owner_renew_raced", `failed to renew DO owner for ${owner.ownerKey}`); + }), DO_OWNERSHIP_CODE.OWNER_RENEW_RACED, `failed to renew DO owner for ${owner.ownerKey}`); } /** @param {DoEnv} env */ @@ -501,7 +606,7 @@ export async function releaseOwner(env, owner) { const key = ownerKeyOf(owner.ownerKey); return await withWatchRetries(async () => client.session(async (session) => { await session.watch(key); - const current = parseOwner(await session.get(key)); + const current = parseOwner(await session.get(key), owner.ownerKey); if (!ownerFenceMatches(current, owner)) { await session.unwatch(); forgetOwnedScope(owner.ownerKey); @@ -510,7 +615,7 @@ export async function releaseOwner(env, owner) { await stageOwnerRelease(session.multi(), key).exec(); forgetOwnedScope(owner.ownerKey); return { released: true, owner: null }; - }), "owner_release_raced", `failed to release DO owner for ${owner.ownerKey}`); + }), DO_OWNERSHIP_CODE.OWNER_RELEASE_RACED, `failed to release DO owner for ${owner.ownerKey}`); } /** @param {DoEnv} env */ diff --git a/do-runtime/protocol.js b/do-runtime/protocol.js index a2bba28..c117df0 100644 --- a/do-runtime/protocol.js +++ b/do-runtime/protocol.js @@ -2,40 +2,87 @@ import { CLASS_NAME_RE, METHOD_NAME_RE, HEADER_NAME_RE, - WORKER_NAME_RE, - NS_FIELD_RE, STORAGE_ID_RE, - VERSION_RE, HOST_ID_RE, DO_HOST_SHARD_COUNT, MAX_ID_BYTES, } from "do-runtime-protocol-wire-grammar"; -import { DoRuntimeError } from "do-runtime-protocol-errors"; -import { hostIdForObject, shardForObjectName } from "do-runtime-protocol-identity"; +import { DoRuntimeError, doErrorResponse } from "do-runtime-protocol-errors"; +import { + hostIdForObject, + isWellFormedUnicodeString, + shardForObjectName, +} from "do-runtime-protocol-identity"; import { formatWorkerId } from "shared-worker-id"; -import { firstWorkerdExperimentalCompatFlag } from "shared-workerd-compat-flags"; import { BodyTooLargeError, readBoundedBytes as readRequestBoundedBytes, readBoundedText as readRequestBoundedText, } from "shared-bounded-body"; import { INTERNAL_AUTH_HEADER } from "shared-internal-auth"; +import { isValidRuntimeLoadNs, WORKER_NAME_RE } from "shared-ns-pattern"; +import { parseVersion } from "shared-version"; export { DO_HOST_SHARD_COUNT } from "do-runtime-protocol-wire-grammar"; export { DoRuntimeError, doErrorResponse } from "do-runtime-protocol-errors"; -export { hostIdForObject, hostIdForShard, shardForObjectName } from "do-runtime-protocol-identity"; +export { + hostIdForObject, + hostIdForShard, + isWellFormedUnicodeString, + shardForObjectName, +} from "do-runtime-protocol-identity"; + +export const DO_OWNERSHIP_CODE = Object.freeze({ + OWNER_CLAIM_RACED: "owner_claim_raced", + OWNER_FENCE_MISSING: "owner_fence_missing", + STALE_OWNER_GENERATION: "stale_owner_generation", + OWNER_LEASE_EXPIRED: "owner_lease_expired", + STALE_OWNER_STORAGE: "stale_owner_storage", + OWNER_LEASE_TOO_SHORT: "owner_lease_too_short", + OWNER_RENEW_RACED: "owner_renew_raced", + OWNER_RELEASE_RACED: "owner_release_raced", + OWNER_UNAVAILABLE: "owner_unavailable", + OWNER_ENDPOINT_MISSING: "owner_endpoint_missing", + FORWARD_HOP_EXHAUSTED: "forward_hop_exhausted", + TASK_DRAINING: "task_draining", +}); +export const DO_OWNERSHIP_ERROR_CONTROL_HEADER = "x-wdl-do-ownership-error"; +/** @type {Set} */ +const DO_OWNERSHIP_CODES = new Set(Object.values(DO_OWNERSHIP_CODE)); + +/** @param {unknown} err */ +export function doPlatformErrorResponse(err) { + const response = doErrorResponse(err); + if (err instanceof DoRuntimeError && DO_OWNERSHIP_CODES.has(err.code)) { + response.headers.set(DO_OWNERSHIP_ERROR_CONTROL_HEADER, err.code); + } + return response; +} -const MAX_MODULE_COUNT = 128; -const MAX_MODULE_SOURCE_BYTES = 1024 * 1024; const MAX_REQUEST_BODY_BYTES = 1024 * 1024; const MAX_INVOKE_ENVELOPE_BYTES = 2 * 1024 * 1024; const MAX_REQUEST_HEADER_COUNT = 128; const MAX_REQUEST_HEADER_BYTES = 64 * 1024; export const DO_INVOKE_CONTENT_TYPE = "application/vnd.wdl.do-invoke"; +const IntrinsicArray = Array; +const IntrinsicNumber = Number; +const IntrinsicObject = Object; +const IntrinsicWeakSet = WeakSet; +const intrinsicReflectApply = Reflect.apply; +const intrinsicArrayIsArray = Array.isArray; +const intrinsicNumberIsFinite = Number.isFinite; +const intrinsicObjectEntries = Object.entries; +const intrinsicObjectGetOwnPropertySymbols = Object.getOwnPropertySymbols; +const intrinsicObjectGetPrototypeOf = Object.getPrototypeOf; +const intrinsicWeakSetAdd = WeakSet.prototype.add; +const intrinsicWeakSetDelete = WeakSet.prototype.delete; +const intrinsicWeakSetHas = WeakSet.prototype.has; const utf8Encoder = new TextEncoder(); const utf8Decoder = new TextDecoder(); const ALARM_INTERNAL_URL = "https://do.internal/__wdl_alarm"; const ALARM_INTERNAL_HEADER = "x-wdl-do-internal-alarm"; +const RPC_INTERNAL_URL = "https://do.internal/__wdl_rpc"; +const RPC_INTERNAL_HEADER = "x-wdl-do-internal-rpc"; const CONNECT_HEADERS = { ns: "x-wdl-do-ns", worker: "x-wdl-do-worker", @@ -58,10 +105,12 @@ const OWNER_HINT_PROTOCOL_HEADERS = [ ]; const CONNECT_INTERNAL_HEADER_NAMES = new Set([ INTERNAL_AUTH_HEADER, + DO_OWNERSHIP_ERROR_CONTROL_HEADER, ...Object.values(CONNECT_HEADERS), ...OWNER_HINT_PROTOCOL_HEADERS, "x-wdl-do-forwarded", "x-wdl-do-hop-count", + RPC_INTERNAL_HEADER, ]); const LOCAL_ACTOR_ENVELOPE_HEADER = "x-wdl-do-local-envelope"; const LOCAL_ACTOR_ENVELOPE_MARKER = "binary"; @@ -70,18 +119,15 @@ const LOCAL_ACTOR_ENVELOPE_MARKER = "binary"; * @typedef {Record} JsonRecord * @typedef {{ ownerKey: string, taskId: string, generation: number }} OwnerFence * @typedef {{ ns: string, worker: string, version: string, doStorageId: string, workerId: string }} BundleSource - * @typedef {{ workerId: string, workerCode: JsonRecord }} InlineWorkerSource - * @typedef {BundleSource | InlineWorkerSource} InvokeSource * @typedef {{ className: string, objectName: string }} ObjectTarget * @typedef {{ method: string, url: string, headers: Array<[string, string]>, bodyBytes?: Uint8Array, bodyBase64?: undefined, bodyText?: undefined }} RequestSpec * @typedef {{ retryCount: number, isRetry: boolean, token?: string }} AlarmInfo * @typedef {{ method: string, args: unknown[] }} RpcInfo - * @typedef {InvokeSource & { kind: "fetch", hostId: string, className: string, objectName: string, props: Record, owner?: OwnerFence | null, request: RequestSpec }} FetchInvoke - * @typedef {InvokeSource & { kind: "alarm", hostId: string, className: string, objectName: string, props: Record, owner?: OwnerFence | null, alarm: AlarmInfo }} AlarmInvoke - * @typedef {InvokeSource & { kind: "rpc", hostId: string, className: string, objectName: string, props: Record, owner?: OwnerFence | null, rpc: RpcInfo }} RpcInvoke + * @typedef {BundleSource & { kind: "fetch", hostId: string, className: string, objectName: string, props: Record, owner?: OwnerFence | null, request: RequestSpec }} FetchInvoke + * @typedef {BundleSource & { kind: "alarm", hostId: string, className: string, objectName: string, props: Record, owner?: OwnerFence | null, alarm: AlarmInfo }} AlarmInvoke + * @typedef {BundleSource & { kind: "rpc", hostId: string, className: string, objectName: string, props: Record, owner?: OwnerFence | null, rpc: RpcInfo }} RpcInvoke * @typedef {FetchInvoke | AlarmInvoke | RpcInvoke} DoInvoke * @typedef {Record & { request?: Record & { bodyBytes?: Uint8Array } }} EnvelopeInvoke - * @typedef {{ allowInlineWorkerCode?: boolean }} NormalizeOptions */ /** @@ -163,6 +209,9 @@ function requireString(value, field, { maxBytes = MAX_ID_BYTES, pattern = null } if (typeof value !== "string" || value.length === 0) { throw new DoRuntimeError(400, "invalid_request", `${field} must be a non-empty string`); } + if (!isWellFormedUnicodeString(value)) { + throw new DoRuntimeError(400, "invalid_request", `${field} must contain well-formed Unicode`); + } if (byteLength(value) > maxBytes) { throw new DoRuntimeError(400, "invalid_request", `${field} is too large`); } @@ -196,35 +245,47 @@ function requireJsonSerializedSize(value, field, maxBytes) { * @param {WeakSet} [seen] * @returns {string | null} */ -function jsonDataError(value, field, seen = new WeakSet()) { +function jsonDataError(value, field, seen = new IntrinsicWeakSet()) { if (value === null) return null; const type = typeof value; if (type === "string" || type === "boolean") return null; - if (type === "number") return Number.isFinite(value) ? null : `${field} must be a finite number`; + if (type === "number") { + return intrinsicReflectApply(intrinsicNumberIsFinite, IntrinsicNumber, [value]) + ? null + : `${field} must be a finite number`; + } if (type === "bigint" || type === "function" || type === "symbol" || type === "undefined") { return `${field} must be JSON data`; } if (type !== "object") return `${field} must be JSON data`; const objectValue = /** @type {Record | unknown[]} */ (value); - if (seen.has(objectValue)) return `${field} must not be circular`; - seen.add(objectValue); - if (Array.isArray(objectValue)) { - for (let i = 0; i < objectValue.length; i++) { - if (!(i in objectValue)) return `${field} must not be sparse`; - const error = jsonDataError(objectValue[i], `${field}[${i}]`, seen); + if (intrinsicReflectApply(intrinsicWeakSetHas, seen, [objectValue])) { + return `${field} must not be circular`; + } + intrinsicReflectApply(intrinsicWeakSetAdd, seen, [objectValue]); + if (intrinsicReflectApply(intrinsicArrayIsArray, IntrinsicArray, [objectValue])) { + const arrayValue = /** @type {unknown[]} */ (objectValue); + for (let i = 0; i < arrayValue.length; i++) { + if (!(i in arrayValue)) return `${field} must not be sparse`; + const error = jsonDataError(arrayValue[i], `${field}[${i}]`, seen); if (error) return error; } - seen.delete(objectValue); + intrinsicReflectApply(intrinsicWeakSetDelete, seen, [objectValue]); return null; } - const proto = Object.getPrototypeOf(objectValue); - if (proto !== Object.prototype && proto !== null) return `${field} must be a plain JSON object`; - if (Object.getOwnPropertySymbols(objectValue).length > 0) return `${field} must not contain symbol keys`; - for (const [key, entry] of Object.entries(objectValue)) { - const error = jsonDataError(entry, `${field}.${key}`, seen); + const proto = intrinsicReflectApply(intrinsicObjectGetPrototypeOf, IntrinsicObject, [objectValue]); + if (proto !== IntrinsicObject.prototype && proto !== null) return `${field} must be a plain JSON object`; + const symbols = intrinsicReflectApply(intrinsicObjectGetOwnPropertySymbols, IntrinsicObject, [objectValue]); + if (symbols.length > 0) return `${field} must not contain symbol keys`; + const entries = /** @type {[string, unknown][]} */ ( + intrinsicReflectApply(intrinsicObjectEntries, IntrinsicObject, [objectValue]) + ); + for (let i = 0; i < entries.length; i++) { + const entry = entries[i]; + const error = jsonDataError(entry[1], `${field}.${entry[0]}`, seen); if (error) return error; } - seen.delete(objectValue); + intrinsicReflectApply(intrinsicWeakSetDelete, seen, [objectValue]); return null; } @@ -346,8 +407,12 @@ function normalizeInvokeKind(value) { */ function normalizeAlarmInfo(value) { const input = value == null ? {} : requireRecord(value, "alarm"); - const retryCount = input.retryCount == null ? 0 : Number(input.retryCount); - if (!Number.isInteger(retryCount) || retryCount < 0) { + const retryCount = input.retryCount == null ? 0 : input.retryCount; + if ( + typeof retryCount !== "number" || + !Number.isInteger(retryCount) || + retryCount < 0 + ) { throw new DoRuntimeError(400, "invalid_request", "alarm.retryCount must be a non-negative integer"); } const token = input.token == null ? undefined : requireString(input.token, "alarm.token"); @@ -390,8 +455,8 @@ function normalizeOwnerFence(value) { if (value == null) return null; const input = requireRecord(value, "owner"); const generation = Number(input.generation); - if (!Number.isInteger(generation) || generation < 0) { - throw new DoRuntimeError(400, "invalid_request", "owner.generation must be a non-negative integer"); + if (!Number.isSafeInteger(generation) || generation <= 0) { + throw new DoRuntimeError(400, "invalid_request", "owner.generation must be a positive safe integer"); } return { ownerKey: requireString(input.ownerKey, "owner.ownerKey"), @@ -421,89 +486,35 @@ function parseHostId(value) { /** * @param {unknown} value - * @param {InvokeSource} source + * @param {BundleSource} source * @param {ObjectTarget} input */ function normalizeHostId(value, source, input) { const parsed = parseHostId(value); if ( - "ns" in source && - ( - parsed.doStorageId !== source.doStorageId || - parsed.className !== input.className || - parsed.shard !== shardForObjectName(input.objectName) - ) + parsed.doStorageId !== source.doStorageId || + parsed.className !== input.className || + parsed.shard !== shardForObjectName(input.objectName) ) { throw new DoRuntimeError(400, "invalid_request", "hostId does not match object shard"); } return parsed.hostId; } -/** - * @param {unknown} value - * @returns {JsonRecord} - */ -function normalizeWorkerCode(value) { - const input = requireRecord(value, "workerCode"); - const modules = requireRecord(input.modules, "workerCode.modules"); - const entries = Object.entries(modules); - if (entries.length === 0) { - throw new DoRuntimeError(400, "invalid_request", "workerCode.modules must not be empty"); - } - if (entries.length > MAX_MODULE_COUNT) { - throw new DoRuntimeError(400, "invalid_request", "workerCode.modules has too many modules"); - } - /** @type {Record} */ - const normalizedModules = {}; - for (const [name, source] of entries) { - const moduleName = requireString(name, "workerCode module name", { maxBytes: 512 }); - if (moduleName.includes("..")) { - throw new DoRuntimeError(400, "invalid_request", "workerCode module names must not contain .."); - } - if (typeof source !== "string") { - throw new DoRuntimeError(400, "invalid_request", `workerCode.modules.${moduleName} must be a string`); - } - if (byteLength(source) > MAX_MODULE_SOURCE_BYTES) { - throw new DoRuntimeError(400, "invalid_request", `workerCode.modules.${moduleName} is too large`); - } - normalizedModules[moduleName] = source; - } - const mainModule = requireString(input.mainModule, "workerCode.mainModule", { maxBytes: 512 }); - if (!Object.hasOwn(normalizedModules, mainModule)) { - throw new DoRuntimeError(400, "invalid_request", "workerCode.mainModule must reference a module"); - } - const compatibilityDate = input.compatibilityDate == null - ? "2026-04-24" - : requireString(input.compatibilityDate, "workerCode.compatibilityDate", { maxBytes: 64 }); - const compatibilityFlags = Array.isArray(input.compatibilityFlags) - ? input.compatibilityFlags.map((flag, index) => requireString(flag, `workerCode.compatibilityFlags[${index}]`, { maxBytes: 128 })) - : ["nodejs_compat"]; - const experimentalFlag = firstWorkerdExperimentalCompatFlag(compatibilityFlags); - if (experimentalFlag) { - throw new DoRuntimeError( - 400, - "experimental_compat_flag_unsupported", - `workerCode.compatibilityFlags contains experimental workerd flag ${JSON.stringify(experimentalFlag)}, which WDL does not support for tenant workers` - ); - } - const env = input.env == null ? {} : requireRecord(input.env, "workerCode.env"); - return { - compatibilityDate, - compatibilityFlags, - mainModule, - modules: normalizedModules, - env, - }; -} - /** * @param {JsonRecord} input * @returns {BundleSource} */ function normalizeBundleSource(input) { - const ns = requireString(input.ns, "ns", { pattern: NS_FIELD_RE }); + const ns = requireString(input.ns, "ns"); + if (!isValidRuntimeLoadNs(ns)) { + throw new DoRuntimeError(400, "invalid_request", "ns is not valid"); + } const worker = requireString(input.worker, "worker", { pattern: WORKER_NAME_RE }); - const version = requireString(String(input.version ?? ""), "version", { pattern: VERSION_RE }); + const version = requireString(input.version, "version"); + if (parseVersion(version) == null) { + throw new DoRuntimeError(400, "invalid_request", "version is not valid"); + } const doStorageId = requireString(input.doStorageId, "doStorageId", { pattern: STORAGE_ID_RE }); return { ns, @@ -516,40 +527,31 @@ function normalizeBundleSource(input) { /** * @param {unknown} value - * @param {NormalizeOptions} [options] * @returns {DoInvoke} */ -export function normalizeDoInvokeRequest(value, options = {}) { +export function normalizeDoInvokeRequest(value) { const input = requireRecord(value, "body"); - const allowInlineWorkerCode = options.allowInlineWorkerCode === true; - const hasInlineWorkerCode = input.workerCode != null; - if (hasInlineWorkerCode && !allowInlineWorkerCode) { - throw new DoRuntimeError(400, "invalid_request", "workerCode is only accepted when DO_TEST_HOOKS=1"); - } - /** @type {InvokeSource} */ - const source = hasInlineWorkerCode - ? { - workerId: requireString(input.workerId, "workerId"), - workerCode: normalizeWorkerCode(input.workerCode), - } - : normalizeBundleSource(input); + if (input.workerCode != null) { + throw new DoRuntimeError(400, "invalid_request", "workerCode is not accepted by the DO invoke protocol"); + } + const source = normalizeBundleSource(input); const kind = normalizeInvokeKind(input.kind); const className = requireString(input.className, "className", { pattern: CLASS_NAME_RE }); const objectName = requireString(input.objectName, "objectName"); const base = { kind, hostId: input.hostId == null - ? defaultHostId(source, { className, objectName }) + ? hostIdForObject(source.doStorageId, className, objectName) : normalizeHostId(input.hostId, source, { className, objectName }), className, objectName, - props: "ns" in source ? { + props: { ns: source.ns, worker: source.worker, version: source.version, doStorageId: source.doStorageId, className, - } : {}, + }, ...(input.owner == null ? {} : { owner: /** @type {OwnerFence} */ (normalizeOwnerFence(input.owner)) }), ...source, }; @@ -591,17 +593,6 @@ export function normalizeDoConnectRequest(request) { return normalizeDoInvokeRequest(body); } -/** - * @param {InvokeSource} source - * @param {ObjectTarget} input - */ -function defaultHostId(source, input) { - if (!("ns" in source) || !source.ns || !source.worker) { - throw new DoRuntimeError(400, "invalid_request", "hostId is required for inline workerCode test hooks"); - } - return hostIdForObject(source.doStorageId, input.className, input.objectName); -} - /** @param {DoInvoke} invoke */ export function buildFacetName(invoke) { return `${invoke.className}:${invoke.objectName}`; @@ -625,6 +616,7 @@ export function buildForwardRequest(spec) { // User-controlled DO fetches must not be able to spoof internal Workflows // alarm delivery; only the alarm builder may attach this. request.headers.delete(ALARM_INTERNAL_HEADER); + request.headers.delete(RPC_INTERNAL_HEADER); request.headers.delete(INTERNAL_AUTH_HEADER); return request; } @@ -689,9 +681,8 @@ export function buildLocalActorRequest(url, invoke, requestId = null) { /** * @param {Request} request - * @param {NormalizeOptions} [options] */ -export async function readLocalActorInvokeRequest(request, options = {}) { +export async function readLocalActorInvokeRequest(request) { if (request.headers.get(LOCAL_ACTOR_ENVELOPE_HEADER) !== LOCAL_ACTOR_ENVELOPE_MARKER) { throw new DoRuntimeError(415, "unsupported_media_type", "DO host actor requests require the local envelope"); } @@ -709,7 +700,7 @@ export async function readLocalActorInvokeRequest(request, options = {}) { } catch { throw new DoRuntimeError(400, "invalid_json", "Request body must be valid JSON"); } - const invoke = normalizeDoInvokeRequest(metadata, options); + const invoke = normalizeDoInvokeRequest(metadata); const bodyBytes = bytes.subarray(4 + metadataLength); if (bodyBytes.length === 0) return invoke; if (!("request" in invoke)) { @@ -745,10 +736,9 @@ function decodeInvokeEnvelope(bytes) { /** * @param {unknown} metadata * @param {Uint8Array} bodyBytes - * @param {NormalizeOptions} options */ -function invokeWithEnvelopeBody(metadata, bodyBytes, options) { - const invoke = normalizeDoInvokeRequest(metadata, options); +function invokeWithEnvelopeBody(metadata, bodyBytes) { + const invoke = normalizeDoInvokeRequest(metadata); if (bodyBytes.length === 0) return invoke; if (!("request" in invoke)) { throw new DoRuntimeError(400, "invalid_request", "DO invoke envelope body is only valid for fetch requests"); @@ -764,9 +754,8 @@ function invokeWithEnvelopeBody(metadata, bodyBytes, options) { /** * @param {Request} request - * @param {NormalizeOptions} [options] */ -export async function readDoInvokeRequest(request, options = {}) { +export async function readDoInvokeRequest(request) { const contentType = request.headers.get("content-type") || ""; if (contentType.split(";", 1)[0].trim().toLowerCase() !== DO_INVOKE_CONTENT_TYPE) { throw new DoRuntimeError( @@ -776,7 +765,7 @@ export async function readDoInvokeRequest(request, options = {}) { ); } const { metadata, bodyBytes } = decodeInvokeEnvelope(await readBoundedBytes(request, MAX_INVOKE_ENVELOPE_BYTES)); - return invokeWithEnvelopeBody(metadata, bodyBytes, options); + return invokeWithEnvelopeBody(metadata, bodyBytes); } /** @param {EnvelopeInvoke} invoke */ @@ -788,14 +777,31 @@ export function encodeDoInvokeRequest(invoke) { }); } -/** @param {JsonRecord} alarm */ -export function buildAlarmRequest(alarm) { +/** @param {JsonRecord} alarm @param {string | null} requestId */ +export function buildAlarmRequest(alarm, requestId) { return new Request(ALARM_INTERNAL_URL, { method: "POST", headers: { "content-type": "application/json", [ALARM_INTERNAL_HEADER]: "1", + ...(requestId ? { "x-request-id": requestId } : {}), }, body: JSON.stringify(alarm), }); } + +/** + * @param {RpcInfo} rpc + * @param {string | null} requestId + */ +export function buildRpcRequest(rpc, requestId) { + return new Request(RPC_INTERNAL_URL, { + method: "POST", + headers: { + "content-type": "application/json", + [RPC_INTERNAL_HEADER]: "1", + ...(requestId ? { "x-request-id": requestId } : {}), + }, + body: JSON.stringify({ method: rpc.method, args: rpc.args }), + }); +} diff --git a/do-runtime/protocol/identity.js b/do-runtime/protocol/identity.js index 6bcf36f..353b63d 100644 --- a/do-runtime/protocol/identity.js +++ b/do-runtime/protocol/identity.js @@ -1,6 +1,7 @@ import { CLASS_NAME_RE, DO_HOST_SHARD_COUNT, + HOST_ID_RE, MAX_ID_BYTES, STORAGE_ID_RE, } from "do-runtime-protocol-wire-grammar"; @@ -8,6 +9,14 @@ import { DoRuntimeError } from "do-runtime-protocol-errors"; import { fnv1a32Utf8 } from "shared-fnv1a32"; const utf8Encoder = new TextEncoder(); +const intrinsicReflectApply = Reflect.apply; +const intrinsicStringIsWellFormed = String.prototype.isWellFormed; + +/** @param {unknown} value */ +export function isWellFormedUnicodeString(value) { + return typeof value === "string" && + intrinsicReflectApply(intrinsicStringIsWellFormed, value, []); +} /** @param {string} value */ function byteLength(value) { @@ -17,12 +26,15 @@ function byteLength(value) { /** * @param {unknown} value * @param {string} field - * @param {RegExp} pattern + * @param {RegExp | null} pattern */ function requireIdentityString(value, field, pattern) { if (typeof value !== "string" || value.length === 0) { throw new DoRuntimeError(400, "invalid_request", `${field} must be a non-empty string`); } + if (!isWellFormedUnicodeString(value)) { + throw new DoRuntimeError(400, "invalid_request", `${field} must contain well-formed Unicode`); + } if (byteLength(value) > MAX_ID_BYTES) { throw new DoRuntimeError(400, "invalid_request", `${field} is too large`); } @@ -32,7 +44,7 @@ function requireIdentityString(value, field, pattern) { throw new DoRuntimeError(400, "invalid_request", `${field} must not contain control characters`); } } - if (!pattern.test(value)) { + if (pattern && !pattern.test(value)) { throw new DoRuntimeError(400, "invalid_request", `${field} is not valid`); } } @@ -42,11 +54,12 @@ function requireIdentityString(value, field, pattern) { * @param {number} [shardCount] */ export function shardForObjectName(objectName, shardCount = DO_HOST_SHARD_COUNT) { + requireIdentityString(objectName, "objectName", null); const count = Number(shardCount); if (!Number.isInteger(count) || count <= 0) { throw new DoRuntimeError(500, "invalid_shard_count", "DO host shard count must be a positive integer"); } - return fnv1a32Utf8(String(objectName)) % count; + return fnv1a32Utf8(objectName) % count; } /** @@ -61,7 +74,9 @@ export function hostIdForShard(doStorageId, className, shard) { } requireIdentityString(doStorageId, "doStorageId", STORAGE_ID_RE); requireIdentityString(className, "className", CLASS_NAME_RE); - return `${doStorageId}:${className}:shard${parsed}`; + const hostId = `${doStorageId}:${className}:shard${parsed}`; + requireIdentityString(hostId, "hostId", HOST_ID_RE); + return hostId; } /** diff --git a/do-runtime/protocol/wire-grammar.js b/do-runtime/protocol/wire-grammar.js index 8d7778b..f36be22 100644 --- a/do-runtime/protocol/wire-grammar.js +++ b/do-runtime/protocol/wire-grammar.js @@ -2,12 +2,6 @@ export const MAX_ID_BYTES = 512; export const CLASS_NAME_RE = /^[A-Za-z_$][A-Za-z0-9_$]*$/; export const METHOD_NAME_RE = /^[A-Za-z_$][A-Za-z0-9_$]*$/; export const HEADER_NAME_RE = /^[!#$%&'*+\-.^_`|~0-9A-Za-z]+$/; -export const WORKER_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,254}$/; -// ns field accepts the same shape gateway does (tenant grammar OR reserved -// ____ form). Without this, DO bindings deployed to reserved ns like -// __system__ fail at every host_invoke / actor_create. -export const NS_FIELD_RE = /^(?:[a-z0-9-]+|__[a-z0-9_-]+__)$/; export const STORAGE_ID_RE = /^[a-z0-9_-]+$/; -export const VERSION_RE = /^v[0-9]+$/; -export const HOST_ID_RE = /^[a-z0-9_-]+:[A-Za-z_$][A-Za-z0-9_$]*:shard[0-9]+$/; +export const HOST_ID_RE = /^[a-z0-9_-]+:[A-Za-z_$][A-Za-z0-9_$]*:shard(?:0|[1-9][0-9]*)$/; export const DO_HOST_SHARD_COUNT = 16; diff --git a/do-runtime/task-identity.js b/do-runtime/task-identity.js index c4a8748..7f393f8 100644 --- a/do-runtime/task-identity.js +++ b/do-runtime/task-identity.js @@ -1,4 +1,5 @@ import { createTaskIdentityResolver } from "shared-task-identity"; +import { validOwnerEndpointForService } from "shared-owner-endpoint"; import { DoRuntimeError } from "do-runtime-protocol"; const resolver = createTaskIdentityResolver({ @@ -8,6 +9,7 @@ const resolver = createTaskIdentityResolver({ serviceLabel: "DO", unavailableCode: "task_identity_unavailable", createError: (status, code, message) => new DoRuntimeError(status, code, message), + validateEndpoint: (endpoint, port) => validOwnerEndpointForService(endpoint, port, "do-runtime"), }); export const { diff --git a/docker-compose.images.yml b/docker-compose.images.yml index 698ce00..8b296a5 100644 --- a/docker-compose.images.yml +++ b/docker-compose.images.yml @@ -1,49 +1,43 @@ +x-rust-published: &rust-published + image: docker.io/getwdl/wdl-rust:latest + pull_policy: always + build: !reset null + +x-workerd-published: &workerd-published + image: docker.io/getwdl/wdl-workerd:latest + pull_policy: always + build: !reset null + services: redis-proxy-user: - image: docker.io/getwdl/wdl-rust:latest - pull_policy: always + <<: *rust-published redis-proxy-system: - image: docker.io/getwdl/wdl-rust:latest - pull_policy: always + <<: *rust-published redis-proxy-do: - image: docker.io/getwdl/wdl-rust:latest - pull_policy: always + <<: *rust-published scheduler: - image: docker.io/getwdl/wdl-rust:latest - pull_policy: always + <<: *rust-published workflows: - image: docker.io/getwdl/wdl-rust:latest - pull_policy: always + <<: *rust-published user-runtime: - image: docker.io/getwdl/wdl-workerd:latest - pull_policy: always + <<: *workerd-published system-runtime: - image: docker.io/getwdl/wdl-workerd:latest - pull_policy: always + <<: *workerd-published gateway: - image: docker.io/getwdl/wdl-workerd:latest - pull_policy: always + <<: *workerd-published d1-runtime: - image: docker.io/getwdl/wdl-workerd:latest - pull_policy: always + <<: *workerd-published d1-runtime-a: - image: docker.io/getwdl/wdl-workerd:latest - pull_policy: always + <<: *workerd-published d1-runtime-b: - image: docker.io/getwdl/wdl-workerd:latest - pull_policy: always + <<: *workerd-published d1-runtime-c: - image: docker.io/getwdl/wdl-workerd:latest - pull_policy: always + <<: *workerd-published do-runtime: - image: docker.io/getwdl/wdl-workerd:latest - pull_policy: always + <<: *workerd-published do-runtime-a: - image: docker.io/getwdl/wdl-workerd:latest - pull_policy: always + <<: *workerd-published do-runtime-b: - image: docker.io/getwdl/wdl-workerd:latest - pull_policy: always + <<: *workerd-published do-runtime-c: - image: docker.io/getwdl/wdl-workerd:latest - pull_policy: always + <<: *workerd-published diff --git a/docker-compose.yml b/docker-compose.yml index 97595da..357ae31 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,6 +18,28 @@ x-internal-auth-env: &internal-auth-env WDL_INTERNAL_AUTH_TOKEN: ${WDL_INTERNAL_AUTH_TOKEN:-local-internal-auth-token} WDL_INTERNAL_AUTH_PREVIOUS_TOKEN: ${WDL_INTERNAL_AUTH_PREVIOUS_TOKEN:-} +x-secret-envelope-env: &secret-envelope-env + SECRET_ENVELOPE_LOCAL_KEY_B64: ${SECRET_ENVELOPE_LOCAL_KEY_B64:-MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=} + SECRET_ENVELOPE_KID: ${SECRET_ENVELOPE_KID:-local:compose:secret-envelope:v1} + +x-redis-proxy-service: &redis-proxy-service + <<: *rust-sidecar-image + command: ["/redis-proxy"] + environment: + <<: [*internal-auth-env, *secret-envelope-env] + REDIS_URL: redis://redis:6379 + DATA_REDIS_URL: redis://redis:6379/1 + REDIS_PROXY_PORT: "7070" + healthcheck: + test: ["CMD", "/redis-proxy", "healthcheck"] + interval: 5s + timeout: 5s + retries: 12 + start_period: 10s + depends_on: + redis: + condition: service_healthy + # D1 runtime variants share local storage, health, entrypoint, and image wiring; # each service block below should only declare its task identity. x-d1-runtime-env: &d1-runtime-env @@ -113,64 +135,13 @@ services: COM_ADOBE_TESTING_S3MOCK_STORE_INITIAL_BUCKETS: wdl-assets,wdl-r2 redis-proxy-user: - <<: *rust-sidecar-image - command: ["/redis-proxy"] - environment: - <<: *internal-auth-env - REDIS_URL: redis://redis:6379 - DATA_REDIS_URL: redis://redis:6379/1 - REDIS_PROXY_PORT: "7070" - SECRET_ENVELOPE_LOCAL_KEY_B64: ${SECRET_ENVELOPE_LOCAL_KEY_B64:-MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=} - SECRET_ENVELOPE_KID: ${SECRET_ENVELOPE_KID:-local:compose:secret-envelope:v1} - healthcheck: - test: ["CMD", "/redis-proxy", "healthcheck"] - interval: 5s - timeout: 5s - retries: 12 - start_period: 10s - depends_on: - redis: - condition: service_healthy + <<: *redis-proxy-service redis-proxy-system: - <<: *rust-sidecar-image - command: ["/redis-proxy"] - environment: - <<: *internal-auth-env - REDIS_URL: redis://redis:6379 - DATA_REDIS_URL: redis://redis:6379/1 - REDIS_PROXY_PORT: "7070" - SECRET_ENVELOPE_LOCAL_KEY_B64: ${SECRET_ENVELOPE_LOCAL_KEY_B64:-MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=} - SECRET_ENVELOPE_KID: ${SECRET_ENVELOPE_KID:-local:compose:secret-envelope:v1} - healthcheck: - test: ["CMD", "/redis-proxy", "healthcheck"] - interval: 5s - timeout: 5s - retries: 12 - start_period: 10s - depends_on: - redis: - condition: service_healthy + <<: *redis-proxy-service redis-proxy-do: - <<: *rust-sidecar-image - command: ["/redis-proxy"] - environment: - <<: *internal-auth-env - REDIS_URL: redis://redis:6379 - DATA_REDIS_URL: redis://redis:6379/1 - REDIS_PROXY_PORT: "7070" - SECRET_ENVELOPE_LOCAL_KEY_B64: ${SECRET_ENVELOPE_LOCAL_KEY_B64:-MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=} - SECRET_ENVELOPE_KID: ${SECRET_ENVELOPE_KID:-local:compose:secret-envelope:v1} - healthcheck: - test: ["CMD", "/redis-proxy", "healthcheck"] - interval: 5s - timeout: 5s - retries: 12 - start_period: 10s - depends_on: - redis: - condition: service_healthy + <<: *redis-proxy-service d1-runtime: <<: *d1-runtime-service @@ -301,7 +272,7 @@ services: command: ["serve", "-b", "/app/dist/workerd-configs/system-runtime-local.bin", "--experimental"] environment: - <<: *internal-auth-env + <<: [*internal-auth-env, *secret-envelope-env] REDIS_ADDR: redis:6379 DATA_REDIS_ADDR: redis:6379 DATA_REDIS_DB: "1" @@ -313,8 +284,6 @@ services: # Server-side env is BOOTSTRAP_TOKEN (auth's infra-managed ops # token); CLI keeps the ADMIN_TOKEN env name as "value to send". BOOTSTRAP_TOKEN: local-dev-token - SECRET_ENVELOPE_LOCAL_KEY_B64: ${SECRET_ENVELOPE_LOCAL_KEY_B64:-MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=} - SECRET_ENVELOPE_KID: ${SECRET_ENVELOPE_KID:-local:compose:secret-envelope:v1} S3_ENDPOINT: http://s3mock:9090 S3_BUCKET: wdl-assets S3_ACCESS_KEY_ID: test @@ -346,6 +315,7 @@ services: scheduler: <<: *rust-sidecar-image command: ["/scheduler"] + stop_grace_period: 30s environment: <<: *internal-auth-env REDIS_URL: redis://redis:6379 @@ -390,6 +360,7 @@ services: workflows: <<: *rust-sidecar-image command: ["/workflows"] + stop_grace_period: 30s environment: <<: *internal-auth-env REDIS_URL: redis://redis:6379 diff --git a/docs/compatibility.md b/docs/compatibility.md index e86b71c..8db661c 100644 --- a/docs/compatibility.md +++ b/docs/compatibility.md @@ -31,7 +31,7 @@ Each row separates four compatibility claims: |---|---|---|---|---|---| | ES module Workers and `fetch()` | Supported | Module evaluation, request dispatch, `Response`/`Request`, service binding JSRPC machinery. | Dynamic `workerLoader`, immutable version ids, wrapper-generated `env`, gateway routing, request logs, and public/private outbound separation. | An uncaught tenant `fetch()` exception maps to platform `502 runtime_error`; exception detail goes to structured logs/live tail, not the client body. | WDL does not emulate every compatibility-date behavior change; workerd is configured with the platform's enabled flags. | | WebSocket upgrade | Supported | WebSocket API and 101 response handling inside workerd. | Gateway `GatewayWsHolder` Durable Object holds public sockets and forwards to runtime/do-runtime so long-lived 101s do not live on ordinary gateway request IoContexts. | WDL optimizes for preserving long-lived gateway-held sockets, while Cloudflare can rely on its global edge session model. | Gateway rolling still drops physical client sockets; application-level resume is not implemented. | -| `compatibility_date` / `compatibility_flags` | Partial | workerd feature flags and compatibility behavior where configured in capnp. | CLI/control stores bundle metadata. Control validates that `compatibility_date` is a real `YYYY-MM-DD` date that is not later than the current UTC date or the maximum date supported by the bundled workerd, and rejects upstream `$experimental` enable flags such as `experimental` / `unsafe_module` before deploy. | WDL treats compatibility metadata as deploy-time platform metadata rather than a complete per-worker historical emulation layer. | No per-worker emulation of every Cloudflare historical behavior; tenant workers cannot opt into upstream experimental-only flags. | +| `compatibility_date` / `compatibility_flags` | Partial | workerd feature flags and compatibility behavior where configured in capnp. | CLI/control stores bundle metadata. Dynamic Workers reject explicit `compatibility_date` values earlier than `2026-04-01`; Control also validates that the value is a real `YYYY-MM-DD` date that is not later than the current UTC date or the maximum date supported by the bundled workerd, rejects upstream `$experimental` enable flags such as `experimental` / `unsafe_module`, rejects `legacy_error_serialization`, and rejects explicit `enhanced_error_serialization` once that behavior is implied by the effective date. | WDL treats compatibility metadata as deploy-time platform metadata rather than a complete per-worker historical emulation layer. | No per-worker emulation of every Cloudflare historical behavior; tenant workers cannot opt into upstream experimental-only flags or disable WDL's required enhanced error serialization. | | `nodejs_compat` | Partial | workerd-provided compatibility when the runtime service has the flag enabled. | CLI carries compatibility flags into metadata. | WDL exposes the workerd-enabled compatibility surface rather than a separate Node.js runtime. | This is not a full Node.js platform contract beyond workerd's enabled surface. | | Python Workers modules | Not supported | workerd has an experimental Python Workers path. | Control rejects `py` module manifests with `python_workers_unsupported`; runtime and do-runtime also fail closed for retained metadata containing `py` modules. | WDL keeps tenant bundles JavaScript/WebAssembly/data only and does not permit cold-load-time Pyodide bootstrap. | Python Workers and mixed JS/Python bundles are unsupported. | @@ -48,7 +48,7 @@ bundled workerd behavior for all dates. | Surface | Status | What workerd provides | Stronger / added in WDL | Different from Cloudflare | Not implemented / gaps | |---|---|---|---|---|---| | KV namespace | Supported | A binding object can be exposed to user code through workerd entrypoint/JSRPC mechanics. | Runtime `KV` facade calls redis-proxy; redis-proxy stores values and metadata in DB 1 hash buckets `kvh:::b:`, with `v:` and `m:` fields, 512-byte key/list-prefix cap, 25 MiB value cap, batch raw-byte budget, TTL/EXAT, and prefix list cursors. | KV storage is Redis-backed and namespace-scoped in a WDL deployment. `cacheTtl` is not a storage freshness contract. | No Cloudflare global edge replication or eventual consistency model. | -| R2 bucket | Supported | Fetch/stream primitives in workerd. | Runtime R2 facade signs S3-compatible requests with platform credentials; CLI parses `[[r2_buckets]]`. | Bucket lifecycle and placement are the S3-compatible backend's responsibility. | `preview_bucket_name` and `jurisdiction` are not supported. | +| R2 bucket | Supported | Fetch/stream primitives in workerd. | Runtime R2 facade signs S3-compatible requests with platform credentials; CLI parses `[[r2_buckets]]`. | Bucket lifecycle and placement are the S3-compatible backend's responsibility. `Headers`-form `httpMetadata` accepts `Expires` only as a canonical IMF-fixdate and rejects malformed values before the host call. | `preview_bucket_name` and `jurisdiction` are not supported. | | ASSETS | Partial | Worker can receive a platform-provided binding object. | CLI uploads assets to S3-compatible storage and runtime exposes the WDL `env.ASSETS.url(path)` helper for tokenized CDN URLs. | WDL assets are an S3/CDN helper model, not Cloudflare Pages asset hosting. | WDL does not provide a full Cloudflare Pages asset pipeline or a fetch-style assets binding contract. | | D1 database | Partial | workerd can host a D1-like binding facade and localDisk-backed SQLite actor code. | WDL implements control-plane metadata, d1-runtime, owner lease/generation fencing, migrations, SQL execution, drain/renew, deploy-time alias freezing, and caps on query bodies, decoded statement payloads, rows, and result bytes before SQLite work or response emission. | D1 storage is WDL-owned SQLite with owner routing, not Cloudflare global D1. Physical SQLite files are not deleted on metadata delete. | No Cloudflare global replication/bookmark semantics. SQLite names under the reserved `_cf_` namespace are rejected by workerd case-insensitively. | | Durable Objects | Partial | Native Durable Object class execution, facet identity, SQLite-backed `ctx.storage.sql`, and in-facet WebSocket hibernation API. | WDL implements runtime facade, do-runtime owner routing, shard leases, Redis generation fencing, alarm shim with Workflows-backed due/retry jobs, gateway-held public WebSocket forwarding, storage ids, and cleanup tombstones. | WDL DO identity, owner routing, and cleanup are WDL-managed rather than Cloudflare migration-compatible. | Same-worker classes only. `script_name`, rename/delete migrations, and platform-level WebSocket session recovery are not implemented. SQLite names under the reserved `_cf_` namespace are rejected by workerd case-insensitively. | diff --git a/docs/compatibility.zh.md b/docs/compatibility.zh.md index bcd10f8..b350476 100644 --- a/docs/compatibility.zh.md +++ b/docs/compatibility.zh.md @@ -24,7 +24,7 @@ |---|---|---|---|---|---| | ES module Workers 和 `fetch()` | Supported | Module evaluation、request dispatch、`Response`/`Request`、service binding JSRPC 机制。 | Dynamic `workerLoader`、immutable version id、wrapper 生成 `env`、gateway routing、request logs、public/private outbound 隔离。 | Tenant `fetch()` 未捕获异常会映射为平台 `502 runtime_error`;异常细节进入结构化日志/live tail,不进客户端 body。 | WDL 不模拟每个 compatibility date 的历史行为;workerd 按平台启用的 flags 运行。 | | WebSocket upgrade | Supported | workerd 内的 WebSocket API 和 101 response 处理。 | Gateway `GatewayWsHolder` Durable Object 承载公网 socket 并转发到 runtime/do-runtime,避免长生命周期 101 留在普通 gateway request IoContext。 | WDL 优先保留 gateway-held 长连接;Cloudflare 依赖自己的全球 edge session 模型。 | Gateway rolling 仍会断开物理 client socket;应用级 resume 未实现。 | -| `compatibility_date` / `compatibility_flags` | Partial | capnp 中启用的 workerd feature flags 和兼容行为。 | CLI/control 保存 bundle metadata。Control 会校验 `compatibility_date` 是真实的 `YYYY-MM-DD` 日期,且不晚于当前 UTC 日期和当前 bundled workerd 支持的最大日期;上游 `$experimental` enable flags(例如 `experimental` / `unsafe_module`)会在 deploy 前被拒绝。 | WDL 把 compatibility metadata 当成部署期平台 metadata,不承诺完整 per-worker 历史行为模拟层。 | 没有 per-worker 模拟所有 Cloudflare 历史行为;tenant worker 不能 opt into 上游 experimental-only flags。 | +| `compatibility_date` / `compatibility_flags` | Partial | capnp 中启用的 workerd feature flags 和兼容行为。 | Dynamic Worker 会拒绝早于 `2026-04-01` 的显式 `compatibility_date`;Control 还会校验该值是真实的 `YYYY-MM-DD` 日期,且不晚于当前 UTC 日期和当前 bundled workerd 支持的最大日期;上游 `$experimental` enable flags(例如 `experimental` / `unsafe_module`)、`legacy_error_serialization`,以及 effective date 已隐式启用该行为时显式声明的 `enhanced_error_serialization` 会在 deploy 前被拒绝。 | WDL 把 compatibility metadata 当成部署期平台 metadata,不承诺完整 per-worker 历史行为模拟层。 | 没有 per-worker 模拟所有 Cloudflare 历史行为;tenant worker 不能 opt into 上游 experimental-only flags,也不能禁用 WDL 要求的 enhanced error serialization。 | | `nodejs_compat` | Partial | runtime service 启用 flag 后由 workerd 提供的兼容 surface。 | CLI 把 compatibility flags 带入 metadata。 | WDL 暴露 workerd 已启用的兼容 surface,而不是单独 Node.js runtime。 | 不承诺超出 workerd 已启用 surface 的完整 Node.js 平台能力。 | | Python Workers modules | Not supported | workerd 有 experimental Python Workers 路径。 | Control 用 `python_workers_unsupported` 拒绝 `py` module manifest;runtime 和 do-runtime 对 retained metadata 中的 `py` module 也 fail closed。 | WDL 保持 tenant bundle 只包含 JavaScript/WebAssembly/data,不允许 cold-load 时触发 Pyodide bootstrap。 | 不支持 Python Workers 和 JS/Python 混合 bundle。 | @@ -35,7 +35,7 @@ Node.js TLS 行为跟随 bundled workerd binary。在 2026-07-01 workerd pin 下 | Surface | 状态 | workerd 提供什么 | WDL 增强 / 新增什么 | 与 Cloudflare 的模型差异 | 未实现 / 缺口 | |---|---|---|---|---|---| | KV namespace | Supported | 通过 workerd entrypoint/JSRPC 机制把 binding object 暴露给用户代码。 | Runtime `KV` facade 调 redis-proxy;redis-proxy 在 DB 1 hash bucket `kvh:::b:` 存 value/metadata,字段为 `v:` 和 `m:`,支持 512-byte key/list-prefix cap、25 MiB value cap、batch raw-byte budget、TTL/EXAT、prefix list cursor。 | KV storage 是 WDL deployment 内的 Redis-backed、namespace-scoped 存储;`cacheTtl` 不是存储新鲜度合同。 | 没有 Cloudflare 全局边缘复制 / eventual consistency 模型。 | -| R2 bucket | Supported | workerd 的 fetch/stream primitives。 | Runtime R2 facade 用平台 credentials 签 S3-compatible request;CLI 解析 `[[r2_buckets]]`。 | Bucket lifecycle/placement 由 S3-compatible backend 负责。 | 不支持 `preview_bucket_name` 和 `jurisdiction`。 | +| R2 bucket | Supported | workerd 的 fetch/stream primitives。 | Runtime R2 facade 用平台 credentials 签 S3-compatible request;CLI 解析 `[[r2_buckets]]`。 | Bucket lifecycle/placement 由 S3-compatible backend 负责。`Headers` 形态的 `httpMetadata` 只接受 canonical IMF-fixdate `Expires`,malformed value 会在 host call 前被拒绝。 | 不支持 `preview_bucket_name` 和 `jurisdiction`。 | | ASSETS | Partial | Worker 可接收平台提供的 binding object。 | CLI 把 assets 上传到 S3-compatible storage;runtime 暴露 WDL 的 `env.ASSETS.url(path)` helper,用于生成 tokenized CDN URL。 | WDL assets 是 S3/CDN helper 模型,不是 Cloudflare Pages asset hosting。 | 不是完整 Cloudflare Pages assets pipeline,也不提供 fetch-style assets binding 合同。 | | D1 database | Partial | workerd 可承载 D1-like facade 和 localDisk-backed SQLite actor 代码。 | WDL 实现 control-plane metadata、d1-runtime、owner lease/generation fencing、migrations、SQL execution、drain/renew、deploy-time alias freezing,并在 SQLite work 或 response emission 前限制 query body、decoded statement payload、row 和 result bytes。 | D1 storage 是 WDL-owned SQLite + owner routing,不是 Cloudflare global D1;metadata delete 不删除物理 SQLite 文件。 | 没有 Cloudflare global replication/bookmark 语义。workerd 会大小写不敏感地拒绝 reserved `_cf_` namespace 下的 SQLite 名称。 | | Durable Objects | Partial | Native Durable Object class execution、facet identity、SQLite-backed `ctx.storage.sql`、facet 内 WebSocket hibernation API。 | WDL 实现 runtime facade、do-runtime owner routing、shard leases、Redis generation fencing、由 Workflows-backed due/retry jobs 驱动的 alarm shim、gateway-held public WebSocket forwarding、storage id、cleanup tombstone。 | WDL DO identity、owner routing 和 cleanup 由 WDL 管理,不兼容 Cloudflare migration 模型。 | 只支持同 worker class;不支持 `script_name`、rename/delete migrations、平台级 WebSocket session recovery。workerd 会大小写不敏感地拒绝 reserved `_cf_` namespace 下的 SQLite 名称。 | diff --git a/docs/modules/cli.md b/docs/modules/cli.md index 90abcd6..cacf5d0 100644 --- a/docs/modules/cli.md +++ b/docs/modules/cli.md @@ -82,7 +82,8 @@ CLI output may display: the value without trying to reconstruct it from package metadata. - `minCliVersion`: the minimum downstream CLI version supported by this platform build. - `urls.control`: the control origin that the request actually reached. -- `urls.namespace`: the tenant namespace origin, returned only for namespace tokens. +- `urls.namespace`: the tenant namespace origin, returned only for namespace tokens when + the platform explicitly configures a public `PLATFORM_DOMAIN`. - `urls.assets`: the configured public assets base URL, returned only when the control plane has a safe absolute `http`/`https` `ASSETS_CDN_BASE`; query and fragment are stripped before returning the hint. @@ -148,7 +149,7 @@ Supported config surfaces: | Field | WDL behavior | |---|---| -| `name`, `main`, `compatibility_date`, `compatibility_flags` | Stored in immutable bundle metadata. Control rejects malformed, future, or bundled-workerd-unsupported `compatibility_date` values before commit; final WorkerCode, including runtime/do-runtime-injected modules and generated workflow keys, must fit workerd's 64 MiB `workerLoader` code limit. | +| `name`, `main`, `compatibility_date`, `compatibility_flags` | Stored in immutable bundle metadata. Control rejects supplied `compatibility_date` values earlier than `2026-04-01`, as well as malformed, future, or bundled-workerd-unsupported values, before commit; final WorkerCode, including runtime/do-runtime-injected modules and generated workflow keys, must fit workerd's 64 MiB `workerLoader` code limit. | | `[vars]` | String, number, and boolean values are accepted and stringified into `env`; vars, namespace/worker secrets, and runtime-injected binding/workflow env values must fit WDL's headroomed workerd 1 MiB `workerLoader` env budget. | | `[[kv_namespaces]]` | `id` is a platform-local KV namespace id, not a Cloudflare UUID. | | `[[r2_buckets]]` | `binding` plus `bucket_name` become a namespace-scoped virtual R2 bucket under the platform S3 bucket. | @@ -171,10 +172,11 @@ would imply platform behavior WDL does not implement. The downstream CLI may expose these surfaces with its own command shape, but the platform-side behavior is fixed: -- Worker listing reads active versions, retained versions, and secret-only entries from - control. -- Worker deletion hard-deletes routes, retained versions, worker secrets, queue - consumers, and crons, then stages asset cleanup after the Redis commit. +- Worker listing reads active versions, retained versions, secret-only entries, and + workflow-definitions-only entries from control. +- Worker deletion hard-deletes routes, retained versions, worker secrets, workflow + definitions, queue consumers, and crons, then stages asset cleanup after the Redis + commit. - Version deletion hard-deletes one retained non-active version. - Secret mutation requires an explicit worker scope or namespace scope. Namespace-wide writes must not be accidental. A submitted empty string is a set secret, not unset. diff --git a/docs/modules/cli.zh.md b/docs/modules/cli.zh.md index 9a78364..ebc1f6c 100644 --- a/docs/modules/cli.zh.md +++ b/docs/modules/cli.zh.md @@ -51,7 +51,7 @@ CLI 可以展示: - `platformVersion`:control 返回的 WDL platform version。Canonical derivation 记录在 `control-auth.zh.md` 的 `/whoami` 段;CLI 应直接展示这个值,不应从 package metadata 自行重建。 - `minCliVersion`:当前 platform build 支持的最低下游 CLI 版本。 - `urls.control`:请求实际到达的 control origin。 -- `urls.namespace`:tenant namespace origin,只对 namespace token 返回。 +- `urls.namespace`:tenant namespace origin,只在 caller 是 namespace token 且 platform 显式配置 public `PLATFORM_DOMAIN` 时返回。 - `urls.assets`:配置的 public assets base URL,只在 control plane 有安全的绝对 `http`/`https` `ASSETS_CDN_BASE` 时返回;返回前会去掉 query 和 fragment。 CLI 应把这些字段当作 diagnostics 和 user-facing guidance 的默认值,而不是替代用户显式配置。如果 `minCliVersion` 大于当前 CLI 版本,CLI 应在执行 mutating command 前 warning 或 fail。可选 URL hint 缺失时,应展示为 unavailable,不应自行猜测。 @@ -91,7 +91,7 @@ WDL 遵循 Wrangler selected-env 继承规则: | 字段 | WDL 行为 | |---|---| -| `name`、`main`、`compatibility_date`、`compatibility_flags` | 存入 immutable bundle metadata。Control 会在 commit 前拒绝格式错误、未来日期或当前 bundled workerd 不支持的 `compatibility_date`;包含 runtime/do-runtime 注入模块和生成 workflow keys 后的最终 WorkerCode 必须落在 workerd 64 MiB `workerLoader` code limit 内。 | +| `name`、`main`、`compatibility_date`、`compatibility_flags` | 存入 immutable bundle metadata。Control 会拒绝早于 `2026-04-01` 的显式 `compatibility_date`,并在 commit 前拒绝格式错误、未来日期或当前 bundled workerd 不支持的值;包含 runtime/do-runtime 注入模块和生成 workflow keys 后的最终 WorkerCode 必须落在 workerd 64 MiB `workerLoader` code limit 内。 | | `[vars]` | 接受 string、number、boolean,并 stringified 进 `env`;vars、namespace/worker secrets、runtime 注入的 binding/workflow env value 必须落在 WDL 留有 headroom 的 workerd 1 MiB `workerLoader` env budget 内。 | | `[[kv_namespaces]]` | `id` 是 platform-local KV namespace id,不是 Cloudflare UUID。 | | `[[r2_buckets]]` | `binding` 加 `bucket_name` 映射为平台 S3 bucket 下的 namespace-scoped virtual R2 bucket。 | @@ -111,8 +111,8 @@ WDL 遵循 Wrangler selected-env 继承规则: 下游 CLI 可以用自己的命令形状暴露这些 surface,但平台侧行为是固定的: -- Worker list 从 control 读取 active version、retained version 和 secret-only entry。 -- Worker deletion hard-delete route、retained version、worker secret、queue consumer 和 cron,并在 Redis commit 后 stage asset cleanup。 +- Worker list 从 control 读取 active version、retained version、secret-only entry 和 workflow-definitions-only entry。 +- Worker deletion hard-delete route、retained version、worker secret、workflow definition、queue consumer 和 cron,并在 Redis commit 后 stage asset cleanup。 - Version deletion hard-delete 一个 retained non-active version。 - Secret mutation 必须显式选择 worker scope 或 namespace scope,避免误写 namespace-wide secret。提交的空字符串是已设置 secret,不是 unset。 - D1 命令管理 namespace D1 database 和 forward-only migration file。Migration filename 是 migration id;已 apply 的文件不应 rename 或编辑。 diff --git a/docs/modules/control-auth.md b/docs/modules/control-auth.md index aeaa0b3..e69ae0e 100644 --- a/docs/modules/control-auth.md +++ b/docs/modules/control-auth.md @@ -14,8 +14,15 @@ admin-host branch. It is not dynamically loaded by workerLoader. Main files: - `control/index.js`: request dispatcher. -- `control/lib.js`: route parser and shared route utilities. -- `control/shared.js`: auth wrapper, JSON responses, Redis singletons, publish helpers. +- `control/lib.js`: pure route/key utilities, bundle metadata parsing, and referrer + shaping. +- `control/shared.js`: auth wrapper, Redis singletons, publish helpers, state-bound + workflow transport wiring, and lifecycle helpers. +- `control/errors.js`, `control/json-body.js`, `control/optimistic.js`: pure Control + response and request-body contracts plus the Control adapter over the shared + optimistic retry loop. +- `control/workflows-client.js`: canonical internal Workflows POST transport with + explicit caller-owned timeout selection. - `control/handlers/*`: endpoint handlers. - `control/routing.js`: promote/reconcile WATCH/MULTI logic. - `auth/index.js`, `auth/lib.js`, `shared/auth-roles.js`: token store, role table, @@ -43,7 +50,7 @@ Worker lifecycle: | Method | Path | Contract | |---|---|---| -| `GET` | `/ns//workers` | Lists workers with namespace-owned state, including deploy-only, active, and secret-only workers. | +| `GET` | `/ns//workers` | Lists workers with namespace-owned state, including deploy-only, active, secret-only, and workflow-definitions-only workers. Each result reports `hasSecrets` and `hasWorkflowDefs`. | | `GET` | `/ns//worker//versions` | Lists retained versions and active status. | | `POST` | `/ns//worker//deploy` | Creates a new immutable version from shorthand code or full module manifest; routes, crons, queue consumers, service refs, platform refs, assets, vars, bindings, and `exports` are version metadata. Python modules and upstream experimental compatibility flags are rejected before commit. | | `POST` | `/ns//worker//promote` | Promotes `{"version":"vN"}` through the WATCH/MULTI routing path. Host declaration failures are 403; live pattern conflicts are 409; exhausted transaction contention is 503. | @@ -96,8 +103,11 @@ Control lifecycle operations are split so each critical transition has one autho hash write; only secret-only workers with no retained versions defer their first load-time budget check to deploy. Secret PUT validates the plaintext size and shape, encrypts it into a `WDL-ENC:` envelope before the Redis mutation/WATCH retry loop, - and reuses the same envelope across retries. Runtime - therefore sees a new immutable version id instead of mutable in-place secret changes. + and reuses the same envelope across retries. Envelope JSON uses the fixed + `v,alg,kid,edek,iv,ct,tag` field order and each base64 field is canonical; Control and + redis-proxy both reject non-canonical persisted forms so malformed direct Redis writes + fail closed. Runtime therefore sees a new immutable + version id instead of mutable in-place secret changes. Secret DELETE removes the target field from the env estimate before decrypting the remaining secret hashes, so deleting the corrupt target can still succeed. Any corrupt remaining namespace or worker secret fails closed; direct Redis repair is not a @@ -109,9 +119,15 @@ Control lifecycle operations are split so each critical transition has one autho active routes, retained versions, service refs, D1 refs, workflow lifecycle checks, queue/cron projections, and delete locks before committing Redis lifecycle deletion. S3 object cleanup is enqueued only after Redis commit succeeds. -- Worker delete lock values and `s3cleanup:` task ids are server-generated random - values. `x-request-id` is diagnostic only and may be reused by clients or retries; it - must not become a lock token or cleanup-task id. +- Worker delete lock values are a `whole:` or `version:` operation kind followed by a + server-generated random token; `s3cleanup:` task ids are also server-generated. + `x-request-id` is diagnostic only and may be reused by clients or retries; it must not + become a lock token or cleanup-task id. Delete, D1 migration, and delegated issue locks + use the token-fenced primitive in `shared/redis-lock.js`; release is token-scoped and + best-effort because TTL expiry bounds a leaked advisory lock and a release error must + not replace the operation's real result. Before a final lifecycle mutation, version + and whole-worker delete refresh the token once, then verify it inside the final WATCH + snapshot so an expired request cannot commit under a replacement holder's lock. - Auth is not a middleware convention. `parseControlRoute()` assigns an action, control sends that action and namespace to auth, and auth evaluates the stored token record against `shared/auth-roles.js`. Dispatcher code should not infer permissions from URL @@ -131,9 +147,11 @@ Control lifecycle operations are split so each critical transition has one autho single `x-forwarded-proto` value of `http` or `https`, `/whoami` uses that protocol for `urls.control` and `urls.namespace`; otherwise it falls back to the request URL protocol seen by control. `urls.namespace` is returned only for tenant namespace - tokens; `urls.assets` is returned only when `ASSETS_CDN_BASE` is a safe absolute - `http`/`https` URL, with query and fragment stripped. The endpoint must not grow - into token list, token lookup, or secret-bearing diagnostics. + tokens when `PLATFORM_DOMAIN` is explicitly configured as a public hostname; Control + does not advertise the internal `workers.local` fallback as a public URL. + `urls.assets` is returned only when `ASSETS_CDN_BASE` is a safe absolute `http`/`https` + URL, with query and fragment stripped. The endpoint must not grow into token list, + token lookup, or secret-bearing diagnostics. - A route shape has an `action` only when method, length, and verb exactly match an authorized shape. Missing action is deliberate: non-ops hit the auth unknown-action red line, while ops can still reach dispatcher/handler path and method validation @@ -157,6 +175,10 @@ Control uses WATCH/MULTI/EXEC for active-version flips, routing changes, delete and lifecycle index updates. Worker lifecycle indexes are authoritative; handlers should not add fallback scans of bundle state. +Secret mutation validates env-var grammar, runtime-reserved names, and reserved +`Object.prototype` keys before persistence so every accepted key can be materialized by +the runtime env builder. + Auth stores token records in Redis and evaluates against `shared/auth-roles.js`. Key families: @@ -175,7 +197,7 @@ Key families: | `ns-hosts:` | Set | Control | Active host reverse index for a namespace. | Promote/reconcile maintains SADD/SREM deltas in the same EXEC. | | `patterns:` | Hash | Control | Pattern-host route slots; values are compact `v2` tab-separated projections. | Reconcile/promote updates and publishes pattern invalidation. | | `worker-version-referrers:::` | Set | Control | Rebuildable service-binding referrer index. | Blocks version delete while callers reference the version. | -| `worker-delete-lock::` | String EX | Control | Per-worker delete critical-section lock. | Expires automatically; execute delete releases by completion. | +| `worker-delete-lock::` | String EX | Control | Per-worker delete critical-section lock; value is `whole:` or `version:`. | Expires automatically; execute delete releases by completion. Only `whole` fences new DO ownership. | | `secrets:`, `secrets::` | Hash | Control | Namespace and worker secret stores; values are `WDL-ENC:` envelopes. Control encrypts writes; redis-proxy decrypts only during `/runtime/load`. | Deleted by secret lifecycle or worker delete. | | `queue:__system__:worker-delete-s3-cleanup:s` | DB 1 Stream | Control/s3-cleanup worker | Best-effort post-commit object cleanup queue; logical queue name is `worker-delete-s3-cleanup`. | Enqueued only after Redis delete commit succeeds; enqueue failure returns `cleanup_queue_failed` warning. | | `auth:token:` | Hash | Auth | Authoritative token record. | Revoke/expiry delete active record and write tombstone fields. | @@ -218,10 +240,11 @@ Auth-specific contract: non-diagnostic action is `auth.delegated_token.issue`. A delegated issue request accepts only `{ template }`; caller-provided `kind`, `ns`, `label`, `expiresAt`, or template allowlist fields are - rejected. Auth re-reads the issuer token record, verifies it is active and has the - template allowlist entry, applies the code-defined Auth template registry, then - writes a target token with `created_by`, `issue_template`, and - `issue_template_version` metadata. Active quota is a live credential capacity guard + rejected. Auth re-reads the issuer token record under the final issue-lock WATCH, + verifies it is active and has the template allowlist entry, applies the code-defined + Auth template registry, then writes a target token with `created_by`, + `issue_template`, and `issue_template_version` metadata. Active quota is a live + credential capacity guard based on `created_by + issue_template + expires_at`; it is not an environment quota. Generated namespaces are ordinary tenant namespace strings embedded in the issued token record. Auth serializes issue for each issuer/template with @@ -258,6 +281,20 @@ Auth-specific contract: - Details may add fields but must not override `error`, `message`, or legacy `reason`. Auth reject reason is the `error` machine code; logs may carry `reason` as diagnostic context. +- `control/errors.js::ControlAbort` is the base in-Control coded abort contract. + Routing and Auth retain their boundary-specific error classes; Deploy may subclass + `ControlAbort` where commit cleanup requires a distinct catch boundary. +- `control/json-body.js` owns bounded Control JSON parsing and its `400`/`413` wire + mapping. `control/optimistic.js` binds strict `WatchError` recognition and Redis + sessions to the retry loop owned by `shared/optimistic-retry.js`. +- `control/lib.js::parseBundleMeta()` is the single parser for persisted bundle + `__meta__`. It requires a JSON object and accepts an error factory so routing, + workflows, delete, deploy, and env-budget paths retain their own catch boundaries. + Absence remains use-site-specific. Paths that need metadata to compute a correct + projection change, uniqueness proof, lifecycle cleanup, workflow view, or environment + budget fail closed while their authoritative route or index still names the bundle. + Deploy discovery/link preflight does not classify absence as `corrupt_meta`; the + watched commit rejects a missing pinned service target as `target_drift`. - Delegated issue 409 reasons have distinct retry meaning: `delegated_issue_busy` is retryable after the issuer/template lock clears, `active_quota_exceeded` is not retryable until existing delegated credentials expire @@ -265,7 +302,8 @@ Auth-specific contract: candidate retry budget. - Control 5xx responses use generic/safe messages. Internal exception text, auth Redis diagnostics, backend messages, and provider errors belong in logs unless the endpoint - explicitly owns a diagnostic response field. + explicitly owns a diagnostic response field. Structured coded-error diagnostic strings + are truncated to 2,048 characters before log emission. - Deploy returns `worker_code_invalid` when final WorkerCode would collide with injected WDL runtime/do-runtime reserved module names or lacks required bundle metadata, and `worker_code_too_large` when final WorkerCode, including runtime/do-runtime-injected @@ -293,8 +331,13 @@ Auth-specific contract: beginning after a scheduler timeout. Each run processes one page, so very large prefixes drain across multiple cron or queue dispatches instead of holding one scheduler dispatch open for minutes. -- Workflow lifecycle blockers are checked through workflows and fail closed on service - errors. +- All Control-to-Workflows internal POSTs use the canonical transport in + `control/workflows-client.js`. Callers retain endpoint-specific timeout, non-2xx, and + response-body interpretation. Workflow management calls and the + unbounded-by-namespace lifecycle delete scan intentionally have no client-side + timeout, matching their pre-consolidation behavior, while DO-alarm cleanup explicitly + retains its existing five-second bound. Workflow lifecycle blockers fail closed on + service errors. - AUTH JSRPC errors or Redis explosions are control-plane failures and map to 503 fail-closed behavior, not tenant-visible authorization fallbacks. @@ -331,6 +374,10 @@ Verify outcomes are logged as success, reject, or error; 5xx outcomes are error - `tests/unit/control-delete-handler.test.js` - `tests/unit/control-deploy-watch.test.js` - `tests/unit/control-lifecycle-indexes.test.js` +- `tests/unit/control-shared-stub.test.js` +- `tests/unit/control-handlers-workflows.test.js` +- `tests/unit/control-d1-migrations.test.js` +- `tests/unit/redis-lock.test.js` - `tests/unit/auth-lib.test.js` - `tests/unit/auth-index.test.js` - `tests/integration/auth-worker.test.js` diff --git a/docs/modules/control-auth.zh.md b/docs/modules/control-auth.zh.md index c23db0a..4ec47e7 100644 --- a/docs/modules/control-auth.zh.md +++ b/docs/modules/control-auth.zh.md @@ -11,8 +11,10 @@ Control 跑在 system-runtime 的 `:8082`,通过 gateway admin-host 分支进 主要文件: - `control/index.js`:request dispatcher。 -- `control/lib.js`:route parser 和共享 route 工具。 -- `control/shared.js`:auth wrapper、JSON response、Redis singleton、publish helper。 +- `control/lib.js`:纯 route/key 工具、bundle metadata parsing 和 referrer shaping。 +- `control/shared.js`:auth wrapper、Redis singleton、publish helper、state-bound workflow transport wiring 和 lifecycle helper。 +- `control/errors.js`、`control/json-body.js`、`control/optimistic.js`:纯 Control response 与 request-body contract,以及 shared optimistic retry loop 的 Control adapter。 +- `control/workflows-client.js`:timeout 由 caller 显式选择的 canonical internal Workflows POST transport。 - `control/handlers/*`:endpoint handlers。 - `control/routing.js`:promote/reconcile WATCH/MULTI 逻辑。 - `auth/index.js`、`auth/lib.js`、`shared/auth-roles.js`:token store、role table、authorization evaluation。 @@ -32,7 +34,7 @@ Worker lifecycle: | Method | Path | 合同 | |---|---|---| -| `GET` | `/ns//workers` | 列出有 namespace-owned state 的 worker,包括 deploy-only、active 和 secret-only worker。 | +| `GET` | `/ns//workers` | 列出有 namespace-owned state 的 worker,包括 deploy-only、active、secret-only 和 workflow-definitions-only worker;每一项都会返回 `hasSecrets` 和 `hasWorkflowDefs`。 | | `GET` | `/ns//worker//versions` | 列出 retained versions 和 active status。 | | `POST` | `/ns//worker//deploy` | 从 shorthand code 或完整 module manifest 创建新的 immutable version;routes、crons、queue consumers、service refs、platform refs、assets、vars、bindings 和 `exports` 都是 version metadata。Python modules 和上游 experimental compatibility flags 会在 commit 前被拒绝。 | | `POST` | `/ns//worker//promote` | 通过 WATCH/MULTI routing path promote `{"version":"vN"}`。Host declaration 失败是 403;live pattern conflict 是 409;transaction contention 耗尽是 503。 | @@ -66,12 +68,12 @@ Control lifecycle 操作会拆开处理,确保每个关键 transition 只有 - Deploy 解析支持的 Wrangler/JSONC 形状,校验 bindings 和 routes,通过 `worker:::next_version` 分配下一个 immutable version,写入 bundle metadata/modules/assets,然后进入 explicit promotion 使用的同一条 promote 路径。分配 version 前,deploy 会按 workerd 64 MiB 上限估算最终 WorkerCode,包含 runtime/do-runtime 注入的 wrapper/client modules 和 workflow import rewrite。真正的 Redis WATCH commit 会在分配真实 version 并完成 metadata materialization(例如 D1 database id 和 workflow keys 解析)之后用 code-budget 和 headroomed `workerLoader` env-budget 做权威检查,然后才写入 version。 - Promote 是唯一 active-route flip。它会 WATCH 本次候选需要的 delete lock、bundle metadata、D1 ref、service-binding target ref、queue consumer key、host declaration 和 pattern key。EXEC 在一个受审计的 transition 中更新 active route、host 反向索引、cron/queue projection、lifecycle index 和 invalidation publication。 -- Secret update/delete 在 worker 有 active route 时,会把 secret-store mutation stage 到 `bumpActiveAndPromote()` 内部,让 budget check、secret hash write、bundle copy 和 route flip 共享同一个 WATCH/MULTI transaction。如果没有 active route,则会在直接写 secret hash 前检查 retained versions 的预算;只有没有 retained versions 的 secret-only worker 会把第一次 load-time budget check 交给 deploy。Secret PUT 会先校验 plaintext 大小和形状,在进入 Redis mutation / WATCH retry loop 前加密成 `WDL-ENC:` envelope,并在重试间复用同一个 envelope。Runtime 因此看到新的 immutable version id,而不是原地可变的 secret。Secret DELETE 会先从 env 估算输入里移除目标字段,再解密剩余 secret hash,所以删除损坏的目标字段仍可成功;但剩余 namespace 或 worker secret 只要损坏就 fail closed,direct Redis repair 不是受支持的一致性路径。Namespace-secret mutation 会 WATCH 需要重新估算的 retained worker/version metadata;如果并发 metadata 变化持续使视图失效,control 返回 `namespace_secret_mutation_contention`。 +- Secret update/delete 在 worker 有 active route 时,会把 secret-store mutation stage 到 `bumpActiveAndPromote()` 内部,让 budget check、secret hash write、bundle copy 和 route flip 共享同一个 WATCH/MULTI transaction。如果没有 active route,则会在直接写 secret hash 前检查 retained versions 的预算;只有没有 retained versions 的 secret-only worker 会把第一次 load-time budget check 交给 deploy。Secret PUT 会先校验 plaintext 大小和形状,在进入 Redis mutation / WATCH retry loop 前加密成 `WDL-ENC:` envelope,并在重试间复用同一个 envelope。Envelope JSON 使用固定的 `v,alg,kid,edek,iv,ct,tag` 字段顺序,各 base64 字段也必须是 canonical shape;Control 与 redis-proxy 都拒绝 non-canonical persisted forms,使 malformed direct Redis writes fail closed。Runtime 因此看到新的 immutable version id,而不是原地可变的 secret。Secret DELETE 会先从 env 估算输入里移除目标字段,再解密剩余 secret hash,所以删除损坏的目标字段仍可成功;但剩余 namespace 或 worker secret 只要损坏就 fail closed,direct Redis repair 不是受支持的一致性路径。Namespace-secret mutation 会 WATCH 需要重新估算的 retained worker/version metadata;如果并发 metadata 变化持续使视图失效,control 返回 `namespace_secret_mutation_contention`。 - Version delete 和 whole-worker delete 都 fail-closed。提交 Redis lifecycle deletion 前,会先收集 active route、retained version、service ref、D1 ref、workflow lifecycle check、queue/cron projection 和 delete lock blocker。S3 object cleanup 只在 Redis commit 成功后 enqueue。 -- Worker delete lock value 和 `s3cleanup:` task id 必须由服务端生成随机值。`x-request-id` 只用于诊断,客户端或重试可能复用它;不能把它用作 lock token 或 cleanup-task id。 +- Worker delete lock value 由 `whole:` 或 `version:` 操作类型加服务端生成的随机 token 组成;`s3cleanup:` task id 也由服务端生成。`x-request-id` 只用于诊断,客户端或重试可能复用它;不能把它用作 lock token 或 cleanup-task id。Delete、D1 migration 和 delegated issue lock 使用 `shared/redis-lock.js` 的 token-fenced primitive;release 按 token 限定且是 best-effort,因为 TTL expiry 会限制 advisory lock 泄漏时间,release error 不能覆盖操作的真实结果。Version delete 和 whole-worker delete 会在最终 lifecycle mutation 前续租一次,并在最终 WATCH snapshot 内验证 token,因此过期请求不能借替代 holder 的 lock 提交。 - Auth 不是一层约定俗成的 middleware。`parseControlRoute()` 分配 action,control 把 action 和 namespace 发给 auth,auth 用存储的 token record 对照 `shared/auth-roles.js` 评估权限。Dispatcher 代码不应自己从 URL prefix 推断权限。 - Delegated token issue 刻意不放进 direct `auth.token.*` lifecycle。`POST /auth/delegated-tokens` 使用 action `auth.delegated_token.issue`,因此 `/auth/tokens` issue/list/revoke 仍保持 strict ops-only;`token-issuer` credential 只能请求 Auth 按一个已配置 template materialize token。 -- `/whoami` 是唯一 namespace-less non-ops action。它对任何有效 token 开放,因为只报告当前 token 自己的 principal、token id、request id 和 public diagnostics。`platformVersion` 是由 bundled workerd date version 派生的 WDL version,使用 `wdl.` namespace,例如 `workerd` `1.20260531.1` 会变成 `wdl.20260531.1`;它不是 project release tag,后者的末尾 counter 可以在同一个 workerd date 上为 WDL-only patch 递增。raw workerd version 不返回。`minCliVersion` 是最低支持的下游 CLI 版本。`urls.control` 是到达 control 的 public origin;当 ingress 提供单个 `x-forwarded-proto` 且值为 `http` 或 `https` 时,`/whoami` 会用该协议生成 `urls.control` 和 `urls.namespace`,否则回退到 control 看到的 request URL 协议。`urls.namespace` 只对 tenant namespace token 返回;`urls.assets` 只在 `ASSETS_CDN_BASE` 是安全的绝对 `http`/`https` URL 时返回,并且会去掉 query 和 fragment。它不能扩展成 token list、token lookup 或携带 secret 的 diagnostics。 +- `/whoami` 是唯一 namespace-less non-ops action。它对任何有效 token 开放,因为只报告当前 token 自己的 principal、token id、request id 和 public diagnostics。`platformVersion` 是由 bundled workerd date version 派生的 WDL version,使用 `wdl.` namespace,例如 `workerd` `1.20260531.1` 会变成 `wdl.20260531.1`;它不是 project release tag,后者的末尾 counter 可以在同一个 workerd date 上为 WDL-only patch 递增。raw workerd version 不返回。`minCliVersion` 是最低支持的下游 CLI 版本。`urls.control` 是到达 control 的 public origin;当 ingress 提供单个 `x-forwarded-proto` 且值为 `http` 或 `https` 时,`/whoami` 会用该协议生成 `urls.control` 和 `urls.namespace`,否则回退到 control 看到的 request URL 协议。`urls.namespace` 只在 caller 是 tenant namespace token 且显式配置了 public hostname 形态的 `PLATFORM_DOMAIN` 时返回;Control 不会把内部 `workers.local` fallback 伪装成 public URL。`urls.assets` 只在 `ASSETS_CDN_BASE` 是安全的绝对 `http`/`https` URL 时返回,并且会去掉 query 和 fragment。它不能扩展成 token list、token lookup 或携带 secret 的 diagnostics。 - 只有 method、path length 和 verb 精确匹配 authorized shape 时,route shape 才带 `action`。缺 action 是有意行为:非 ops 会触发 auth unknown-action red line,ops 仍可到 dispatcher/handler 的 path 和 method validation,而不是被 auth 拦下。`/reload` 这类已知 top-level route 的 wrong-method case 会在 ops auth 通过后返回 `405 method_not_allowed`。 ## Redis / Storage 合同 @@ -88,6 +90,8 @@ Control 拥有 DB 0 metadata: Control 对 active-version flip、routing change、delete lock 和 lifecycle index update 使用 WATCH/MULTI/EXEC。Worker lifecycle index 是权威状态;handler 不应增加扫描 bundle state 的 fallback。 +Secret mutation 在写入前校验 env-var grammar、runtime-reserved name 和保留的 `Object.prototype` key,确保每个被接受的 key 都能由 runtime env builder materialize。 + Auth 在 Redis 中存 token record,并按 `shared/auth-roles.js` 评估权限。 Key families: @@ -106,7 +110,7 @@ Key families: | `ns-hosts:` | Set | Control | namespace 的 active host 反向索引。 | promote/reconcile 在同一个 EXEC 内维护 SADD/SREM delta。 | | `patterns:` | Hash | Control | pattern-host route slots;value 是紧凑 `v2` tab-separated projection。 | reconcile/promote 更新并 publish pattern invalidation。 | | `worker-version-referrers:::` | Set | Control | 可重建 service-binding referrer index。 | caller 仍引用该 version 时阻止 version delete。 | -| `worker-delete-lock::` | String EX | Control | per-worker delete critical-section lock。 | 自动过期;execute delete 完成后释放。 | +| `worker-delete-lock::` | String EX | Control | per-worker delete critical-section lock;value 是 `whole:` 或 `version:`。 | 自动过期;execute delete 完成后释放;只有 `whole` 会阻止新的 DO ownership。 | | `secrets:`、`secrets::` | Hash | Control | namespace 和 worker secret store;value 是 `WDL-ENC:` envelope。Control 写入前加密;redis-proxy 只在 `/runtime/load` 时解密。 | secret lifecycle 或 worker delete 删除。 | | `queue:__system__:worker-delete-s3-cleanup:s` | DB 1 Stream | Control / s3-cleanup worker | best-effort post-commit object cleanup queue;逻辑队列名是 `worker-delete-s3-cleanup`。 | 只在 Redis delete commit 成功后 enqueue;enqueue 失败返回 `cleanup_queue_failed` warning。 | | `auth:token:` | Hash | Auth | token record 权威记录。 | revoke/expiry 删除 active record 并写 tombstone 字段。 | @@ -123,7 +127,7 @@ Auth 子合同: - Reserved namespace check 同时存在于 route/auth red line 和 role scope check;两者不能互相替代。 - `issue` 可以签发 `ns`、`platform`、`platform-observer` 和 `ops-observer` token。`token-issuer` 也可由 ops direct issue,但必须提供 camelCase `issueTemplates`,且其中每个 id 都要指向已有 delegated issue template。Auth 把这个 allowlist 存成 Redis `issue_templates` JSON array string,并拒绝 public `issue_templates` 输入,避免 API shape 和 storage shape 混用。`ops` 只能来自 bootstrap。Token plaintext 只生成一次、展示一次,并且只以 SHA-256 hash 形式存储。 - 内置 delegated issue template 包括用于 workshop pool 的 `wdl-chat-ns-pool`,以及用于 hosted CLI live integration 短期 namespace 的 `wdl-cli-integration`。 -- `token-issuer` 是 unbound role;除 `/whoami` self-introspection 外,它唯一 non-diagnostic action 是 `auth.delegated_token.issue`。Delegated issue 请求只接受 `{ template }`;caller-provided `kind`、`ns`、`label`、`expiresAt` 或 template allowlist 字段都会被拒。Auth 会重新读取 issuer token record,确认它仍 active 且 allowlist 包含该 template,再应用 Auth 代码内定义的 template registry,并写入带 `created_by`、`issue_template`、`issue_template_version` metadata 的目标 token。Active quota 基于 `created_by + issue_template + expires_at` 计算,是 live credential capacity guard,不是 environment quota。生成的 namespace 是写入 token record 的普通 tenant namespace 字符串。Auth 会先用 `auth:delegated-issue-lock::` 串行化同一 issuer/template 的发放和 quota 计数,然后拒绝已经出现在 control 维护的 `namespaces` set 或任意已扫描 token record stored `ns` 中的 generated namespace candidate,不区分 token kind、issuer、template、是否过期或是否撤销。Quota 计算 fail-closed:同一 issuer/template 的 delegated token record 损坏会阻止新的 delegated issue,直到 storage contract violation 被修复。`GET /auth/tokens` 是 operator repair surface;单条 malformed `issue_templates` 会作为 invalid entry 列出,不会拖垮整个 list。V1 的 delegated namespace collision check 是 best-effort:unbound/full-plane token 仍能执行不会留下 auth-visible `ns` token record 的 namespace-scoped 写入,而 `namespaces` 只表示 active worker。常规 delegated namespace workflow 应使用 namespace-bound credential 做 namespace-scoped 写;未来的持久 namespace fact index 才是这个 residual risk 的完整修复。 +- `token-issuer` 是 unbound role;除 `/whoami` self-introspection 外,它唯一 non-diagnostic action 是 `auth.delegated_token.issue`。Delegated issue 请求只接受 `{ template }`;caller-provided `kind`、`ns`、`label`、`expiresAt` 或 template allowlist 字段都会被拒。Auth 会在最终 issue-lock WATCH 内重新读取 issuer token record,确认它仍 active 且 allowlist 包含该 template,再应用 Auth 代码内定义的 template registry,并写入带 `created_by`、`issue_template`、`issue_template_version` metadata 的目标 token。Active quota 基于 `created_by + issue_template + expires_at` 计算,是 live credential capacity guard,不是 environment quota。生成的 namespace 是写入 token record 的普通 tenant namespace 字符串。Auth 会先用 `auth:delegated-issue-lock::` 串行化同一 issuer/template 的发放和 quota 计数,然后拒绝已经出现在 control 维护的 `namespaces` set 或任意已扫描 token record stored `ns` 中的 generated namespace candidate,不区分 token kind、issuer、template、是否过期或是否撤销。Quota 计算 fail-closed:同一 issuer/template 的 delegated token record 损坏会阻止新的 delegated issue,直到 storage contract violation 被修复。`GET /auth/tokens` 是 operator repair surface;单条 malformed `issue_templates` 会作为 invalid entry 列出,不会拖垮整个 list。V1 的 delegated namespace collision check 是 best-effort:unbound/full-plane token 仍能执行不会留下 auth-visible `ns` token record 的 namespace-scoped 写入,而 `namespaces` 只表示 active worker。常规 delegated namespace workflow 应使用 namespace-bound credential 做 namespace-scoped 写;未来的持久 namespace fact index 才是这个 residual risk 的完整修复。 - `expiresAt` 必须是严格 ISO-8601 UTC、毫秒精度,并通过真实日历 round trip 校验。过期 token 在过期后第一次 verify 时 lazy collect:删除 active hash index,并写入 `expired_at`;主动 revoke 使用 `revoked_at`。 - 保留 token id `bootstrap` 是 infra-managed ops token。`BOOTSTRAP_TOKEN` 在 auth cold start 时 upsert,`revoke("bootstrap")` 被禁止,轮换方式是更新环境变量并 redeploy。Verify 会缓存已 ensure 的 bootstrap hash,但 Redis 丢失后可以重新 ensure,因此 bootstrap token 能恢复被 flush 的 auth store。 @@ -133,14 +137,17 @@ Auth 子合同: - 如果 route shape 没有 action,非 ops 在 auth 处 fail closed;ops 可以到 dispatcher/handler 的 path 和 method validation;已知 top-level wrong-method route 返回 `405 method_not_allowed`。 - Control JSON error 形状是 `{ error, message }`;details 只能 additive。 - Details 可以增加字段,但不能覆盖 `error`、`message` 或 legacy `reason`。Auth reject reason 是 `error` machine code;日志可以把 `reason` 作为诊断上下文。 +- `control/errors.js::ControlAbort` 是 Control tier 内的基础 coded abort contract。Routing 和 Auth 保留各自 boundary-specific error class;Deploy 可以在 commit cleanup 需要独立 catch boundary 时 subclass `ControlAbort`。 +- `control/json-body.js` 持有 bounded Control JSON parsing 及其 `400`/`413` wire mapping;`control/optimistic.js` 把 strict `WatchError` recognition 和 Redis session 绑定到 `shared/optimistic-retry.js` 持有的 retry loop。 +- `control/lib.js::parseBundleMeta()` 是 persisted bundle `__meta__` 的唯一 parser。它要求值是 JSON object,并通过 error factory 让 routing、workflows、delete、deploy 和 env-budget 保留各自的 catch boundary。Bundle 缺失的语义仍由具体 use site 持有:当 projection 变更、唯一性证明、lifecycle cleanup、workflow view 或 environment budget 必须消费 metadata 才能产生正确结果时,只要权威 route 或 index 仍指向该 bundle,缺失就 fail closed。Deploy discovery/link preflight 不把缺失归类为 `corrupt_meta`;watched commit 会以 `target_drift` 拒绝缺失的 pinned service target。 - Delegated issue 的 409 reason 有不同 retry 语义:`delegated_issue_busy` 表示 issuer/template lock 清除后可重试;`active_quota_exceeded` 在已有 delegated credential 过期或 revoke 前不应重试;`namespace_collision` 表示 Auth 已耗尽 configured candidate retry budget。 -- Control 5xx response 使用 generic/safe message。Internal exception text、auth Redis diagnostic、backend message 和 provider error 应进入日志;除非 endpoint 明确拥有某个 diagnostic response field,否则不进入客户端 body。 +- Control 5xx response 使用 generic/safe message。Internal exception text、auth Redis diagnostic、backend message 和 provider error 应进入日志;除非 endpoint 明确拥有某个 diagnostic response field,否则不进入客户端 body。Structured coded-error diagnostic string 在写入日志前截断为最多 2,048 个字符。 - Deploy 在最终 WorkerCode 会碰撞 WDL 注入的 runtime/do-runtime 保留模块名或缺少必要 bundle metadata 时返回 `worker_code_invalid`,在最终 WorkerCode(包含 runtime/do-runtime 注入模块和生成的 workflow keys)超过 workerd 64 MiB dynamic code limit 时返回 `worker_code_too_large`。Deploy 和 secret mutation 在估算的 `workerLoader` env 超过 WDL headroomed 1 MiB budget 时返回 `worker_env_too_large`。`worker_env_too_large` details 包含 `namespace`、可选 `worker`、`env_bytes`、`max_env_bytes`、`upstream_max_env_bytes` 和 `headroom_bytes`。Deploy 阶段的逐版本检查还会包含 `version`。Secret mutation 重新估算既有 version 时,details 还会包含 `source_version` 和 `estimated_version`。`source_version` 是 operator 应检查、删除或 redeploy 的已存版本;`estimated_version` 是本次预算估算使用的 tag,在 worker-secret bump 路径中就是已分配的精确 bump version。 - Control 不直接调用 gateway。它写 Redis 并 publish invalidation message。 - Control 在进入 Redis mutation loop 前加密 secret PUT value。加密/provider 失败会返回 control error,不写 plaintext fallback。 - Worker delete 先 commit Redis lifecycle state;异步 S3 cleanup enqueue 是 best-effort,失败时返回 warning。 - `s3-cleanup` system worker 会把 cleanup task 持久化在 D1;row 存在后由 cron replay 负责重试。S3 失败使用分钟级 exponential backoff,最高 30 分钟;大前缀 cleanup 每完成一个 S3 List/Delete page 就 checkpoint continuation token,因此正常分页进度不会消耗 failure attempts,也不会在 scheduler timeout 后从头开始。每次 run 只处理一页,所以超大 prefix 会跨多个 cron 或 queue dispatch 排空,而不是让单次 scheduler dispatch 持续数分钟。 -- Workflow lifecycle blocker 通过 workflows 检查;服务错误时 fail closed。 +- 所有 Control-to-Workflows internal POST 都使用 `control/workflows-client.js` 的 canonical transport。Caller 继续持有 endpoint-specific timeout、non-2xx 和 response-body interpretation。Workflow 管理调用和按 namespace 大小无上界的 lifecycle delete scan 都刻意不设 client-side timeout,保持 transport 收敛前的语义;DO-alarm cleanup 则显式保留原有五秒边界。Workflow lifecycle blocker 在服务错误时 fail closed。 - AUTH JSRPC 错误或 Redis 爆炸属于控制面失败,映射为 503 fail closed,而不是 tenant-visible authorization fallback。 ## 安全边界 @@ -170,6 +177,10 @@ Verify outcome 记录为 success、reject 或 error;5xx outcome 是 error log - `tests/unit/control-delete-handler.test.js` - `tests/unit/control-deploy-watch.test.js` - `tests/unit/control-lifecycle-indexes.test.js` +- `tests/unit/control-shared-stub.test.js` +- `tests/unit/control-handlers-workflows.test.js` +- `tests/unit/control-d1-migrations.test.js` +- `tests/unit/redis-lock.test.js` - `tests/unit/auth-lib.test.js` - `tests/unit/auth-index.test.js` - `tests/integration/auth-worker.test.js` diff --git a/docs/modules/d1.md b/docs/modules/d1.md index b616898..7a441c0 100644 --- a/docs/modules/d1.md +++ b/docs/modules/d1.md @@ -66,6 +66,8 @@ Control metadata is explicit: - Delete writes tombstones and does not physically remove SQLite files. - Deploy input may use an alias or physical id; control freezes physical `databaseId` and display `databaseName` into bundle metadata. +- Runtime materialization, query decoding, and rebalance revalidate namespace and + physical `databaseId` against the shared canonical grammar. Runtime ownership: @@ -162,7 +164,13 @@ failover. - Tenant code does not receive backend endpoints or owner credentials. - Tenant-visible D1 metadata and errors must not include owner task ids, backend endpoints, or raw transport error text. -- D1 owner hints must pass service-specific endpoint grammar and generation/task checks. +- D1 owner hints must pass service-specific endpoint grammar and carry a task id plus a + positive JavaScript-safe-integer generation. +- Task identities, rebalance targets, and persisted owner records are validated on write + and read. A persisted record must reconstruct the physical database key under which it + was read; records stored under a different database scope fail closed. Owner forwarding + accepts only the D1 service/headless DNS forms or private RFC1918/100.64 IPv4 addresses + on port 8787; invalid records fail closed before internal auth is attached. - Test hooks are opt-in and must not be enabled in live-ready services. ## Observability diff --git a/docs/modules/d1.zh.md b/docs/modules/d1.zh.md index 2f4623f..f46fc04 100644 --- a/docs/modules/d1.zh.md +++ b/docs/modules/d1.zh.md @@ -39,6 +39,7 @@ Control metadata 是显式 lifecycle: - 用户可见路径只接受 ready database。 - Delete 写 tombstone,不物理删除 SQLite 文件。 - Deploy input 可使用 alias 或 physical id;control 将 physical `databaseId` 和显示用 `databaseName` 冻结进 bundle metadata。 +- Runtime materialization、query decoding 和 rebalance 使用共享 canonical grammar 重新校验 namespace 与 physical `databaseId`。 Runtime ownership: @@ -88,7 +89,8 @@ Key families: - D1 binding metadata 由 control 在 deploy time 冻结。 - Tenant code 不获得 backend endpoint 或 owner credential。 - Tenant-visible D1 metadata 和 error 不得包含 owner task id、backend endpoint 或原始 transport error 文本。 -- D1 owner hint 必须通过 service-specific endpoint grammar 和 generation/task 检查。 +- D1 owner hint 必须通过 service-specific endpoint grammar,并携带 task id 和正的 JavaScript-safe-integer generation。 +- Task identity、rebalance target 和 persisted owner record 在写入和读取时都会校验。Persisted record 必须能重建出读取它的 physical database key;错放在其他 database scope 下的记录会 fail closed。Owner forwarding 只接受 8787 端口上的 D1 service/headless DNS,或 RFC1918/100.64 私网 IPv4;非法记录在附加 internal auth 前 fail closed。 - Test hooks 是 opt-in,不应在 live-ready service 中开启。 ## 可观测性 diff --git a/docs/modules/durable-objects.md b/docs/modules/durable-objects.md index 27c48d1..f49dfda 100644 --- a/docs/modules/durable-objects.md +++ b/docs/modules/durable-objects.md @@ -64,6 +64,26 @@ partial batch result; that is a result envelope, not a generic JSON error envelo Tenant-originated DO fetch bodies are capped at 1 MiB in the runtime facade. The facade rejects an oversized `Content-Length` before reading, and streamed bodies are read incrementally so the cap is enforced before buffering the full body. +DO RPC method names use the JavaScript identifier grammar and are capped at 256 ASCII +bytes by both the runtime client and do-runtime protocol reader. RPC arguments are +structural JSON data capped at 1 MiB: finite numbers, strings, booleans, null, dense +arrays, and plain objects are accepted. Serialization does not invoke `toJSON()` hooks; +sparse arrays, circular structures, non-plain objects, and non-JSON values fail before +dispatch. +do-runtime invokes tenant alarm and RPC methods through private fetch dispatches +intercepted by the generated wrapper. Those requests carry the outer request id so the +host facade establishes invocation-local context without adding platform metadata to +the tenant argument list. Nested DO fetch/connect requests discard tenant-supplied +`x-request-id` values and, when the ambient parent context is available, propagate its +sanitized request id. Request ids remain untrusted diagnostic metadata. +DO invoke envelopes identify persisted bundles by canonical namespace, worker, version, +and storage id. They do not accept inline worker source. +Tenant-facing DO object names and ids must be well-formed Unicode strings. Lone UTF-16 +surrogates are rejected by `idFromName()` / `idFromString()` before hashing or dispatch; +do-runtime alarm ingress and Workflows revalidation enforce the same boundary. +DO host ids are capped at 512 UTF-8 bytes and use canonical `shardN` suffixes without +leading zeroes. DO binding class names use the ASCII JavaScript class-name grammar and +are capped at 468 bytes at deploy, so every shard suffix fits the aggregate host-id cap. ## Redis / Storage Contracts @@ -110,6 +130,9 @@ when the logical worker is gone or now points at a different `doStorageId`. - One task owns a class shard at a time. - Generation fencing prevents stale owners from committing after ownership moves. +- `do-runtime/protocol.js` owns the DO ownership error vocabulary. The injected runtime + transport keeps its retry and stale-hint subsets private and pins them through + response-classification tests. - Facet identity is `className:objectName` inside stable `doStorageId`, so worker promotion preserves object state. - Existing native facets keep the constructed class version until host actor restart or @@ -131,17 +154,25 @@ when the logical worker is gone or now points at a different `doStorageId`. Client messages queued under an older backend reconnect epoch may be discarded without per-frame ack/nack when the gateway resets that epoch. - Ordinary fetch/RPC can fall back through the router after explicit stale-owner or - owner-race responses. A direct owner transport failure, or a 502/503/504 without a - fresh owner-hint header, evicts the cached hint. Safe `GET`/`HEAD` requests may replay - through the router to rediscover the owner; non-idempotent methods and RPC return - `owner_unavailable` without replay because the owner may already have applied the - request. + owner-race responses carrying do-runtime's private ownership-error control header. + Tenant response bodies cannot opt into replay. A direct owner transport failure, or a + 502/503/504 without a fresh owner-hint header, evicts the cached hint. Safe `GET`/`HEAD` + requests may replay through the router to rediscover the owner; non-idempotent methods + and RPC return `owner_unavailable` without replay because the owner may already have + applied the request. +- The shared runtime transport owns owner-hint cache wiring, invoke race retry, and + response-header stripping for both the host binding and injected facade. Its connect + wrapper deliberately omits the invoke-only router fallback required to preserve + owner-established WebSocket upgrades. - `WEBSOCKET_RECONNECT_DELAYS_MS` and `WEBSOCKET_MAX_BUFFERED_MESSAGES` tune gateway backend reconnect budget and client-message buffering without a code rebuild. - Alarm delivery is at-least-once. Scheduler wakes Workflows; Workflows promotes due internal alarm jobs to ready, claims one job under a DB 2 run token, and calls do-runtime `/internal/do/alarms/dispatch`. do-runtime still constructs the native `DoInvoke{kind:"alarm"}` request and uses the normal owner router/fence path. +- Alarm mutation, retarget, dispatch, and whole-worker storage cleanup accept only the + canonical positive JavaScript-safe-integer worker version grammar. Invalid internal or + persisted versions fail before a job is stored or a worker invoke is attempted. - Alarm due times are Unix millisecond timestamps supplied to `setAlarm()`. Workflows and do-runtime both evaluate those timestamps with their local wall clocks; if a backend ready hint reaches do-runtime before the SQLite alarm row is locally due, @@ -165,8 +196,12 @@ when the logical worker is gone or now points at a different `doStorageId`. Owner resolution is the single-writer protocol: 1. do-runtime derives an owner scope from `doStorageId`, class name, and shard. -2. It WATCHes the owner record, generation key, and active worker storage pointer before - claiming or renewing. +2. Owner resolution WATCHes the owner record, generation key, worker delete lock, and + active worker storage pointer. A `whole` delete lock rejects ownership; a `version` + lock remains part of the watched snapshot but does not interrupt active storage. The + WATCH prevents a claim from committing after whole-worker delete starts. Renewal + separately WATCHes the owner record and active storage pointer; its generation fence + is carried by the owner record rather than a second generation-key read. 3. If a live owner exists on another task, the router returns that owner or an owner-hint header; the runtime facade may retry directly, but the owner task still rechecks the fence. @@ -203,15 +238,34 @@ interrupt. ## Security Boundaries -- do-runtime internal endpoints are private-mesh only and are not - application-authenticated. +- do-runtime internal endpoints are private-mesh only and require the shared + `WDL_INTERNAL_AUTH_TOKEN` through `x-wdl-internal-auth`; health and metrics are the + only unauthenticated endpoints. - Tenant code reaches DOs only through runtime-generated facades and frozen metadata. - Tenant-visible DO metadata and errors must not include owner task ids, backend endpoints, or raw transport error text. - Owner hints are trusted only when returned by do-runtime headers and validated against - endpoint grammar. -- Owner-hint defense is layered: tenant response bodies are ignored, only do-runtime - control headers are trusted, and endpoint grammar/acceptable-address checks must pass. + endpoint grammar. Owner hints and invoke fences must carry a positive + JavaScript-safe-integer generation. +- Task identities and persisted owner records are validated on write and read. A + persisted record's `ownerKey`, `hostId`, storage id, class, and shard must reconstruct + the Redis scope under which it was read. During owner resolution, its canonical + namespace and worker must also match the invoking bundle before do-runtime reads that + bundle's active storage pointer. Owner forwarding accepts only the DO service/headless + DNS forms or private RFC1918/100.64 IPv4 addresses on port 8788; invalid records fail + closed before internal auth is attached. +- Owner-hint and ownership-error defense is layered: tenant response bodies and + tenant-supplied control headers are ignored, only do-runtime control headers are + trusted, and endpoint grammar/acceptable-address checks must pass for hints. +- The injected DO transport and shared D1/DO endpoint validator evaluate before tenant + modules and capture the intrinsics used for private-header stripping, request bounds, + invoke serialization, replay classification, and endpoint validation. Tenant prototype + mutation cannot rewrite a trusted target or replay policy after those checks. +- The injected alarm shim also evaluates before the tenant module and captures the + request, response, numeric, proxy, and reflection operations that classify internal + alarms, update their SQLite state, and install the storage facade. Tenant top-level + mutation of those intrinsics cannot redirect an internal alarm to the tenant fetch + handler or prevent that facade installation. - do-runtime supervisor must call local `127.0.0.1:8788` drain/renew endpoints; Service Connect aliases may hit a different task. @@ -267,8 +321,11 @@ recovery after the initial 101. fenced to the deleted `doStorageId`, so a same-name redeploy with a new storage id is not swept by the old delete. If best-effort cleanup fails, a far-future residual alarm job can remain in DB 2 until it becomes due; it then self-discards because the storage - pointer is gone. - `do:objects:` remains a tombstone for future platform cleanup. + pointer is gone. First owner claim watches the same per-worker delete lock and storage + pointer; only the `whole` lock kind rejects ownership, so deleting an inactive version + does not interrupt the active worker. A whole-worker delete therefore cannot miss + owner/generation state created after its final owner scan. `do:objects:` + remains a tombstone for future platform cleanup. - DO object registry writes are best-effort. Dispatch continues if the registry write fails, so the tombstone set may be incomplete; future cleanup must tolerate missing members and treat the active storage pointer plus owner/alarm state as the stronger diff --git a/docs/modules/durable-objects.zh.md b/docs/modules/durable-objects.zh.md index 7f1e9f7..d0a542b 100644 --- a/docs/modules/durable-objects.zh.md +++ b/docs/modules/durable-objects.zh.md @@ -35,6 +35,11 @@ Storage cleanup endpoint 是 native facet storage cleanup 和 worker storage cle DO protocol error 使用 `{ error, message, details? }`。不同于 admin HTTP 的 flat additive error shape,DO protocol detail 会嵌套在 `details` 下,因为消费者是 runtime/DO client protocol,不是通用 admin JSON parser。未知 internal exception 仍会降级成安全的 `internal_error` / `Internal error` message。Storage delete-worker 在 partial batch result 时可能返回 HTTP 207 和 `{ ok:false, deleted, errors }`;这是 result envelope,不是 generic JSON error envelope。 Tenant-originated DO fetch body 在 runtime facade 中限制为 1 MiB。Facade 会在读取前拒绝超限的 `Content-Length`,streamed body 会增量读取,因此 limit 会在完整 buffering 前生效。 +DO RPC method name 使用 JavaScript identifier grammar,并由 runtime client 和 do-runtime protocol reader 同时限制为最多 256 ASCII bytes。RPC 参数是最多 1 MiB 的 structural JSON data:接受 finite number、string、boolean、null、dense array 和 plain object。序列化不会调用 `toJSON()` hook;sparse array、circular structure、non-plain object 和 non-JSON value 会在 dispatch 前失败。 +do-runtime 会通过 generated wrapper 截获的私有 fetch dispatch 调用 tenant alarm 和 RPC method;这些 request 携带外层 request id,使 host facade 建立 invocation-local context,且不会把平台 metadata 加入 tenant argument list。嵌套 DO fetch/connect request 会丢弃 tenant 提供的 `x-request-id`,并在 ambient parent context 可用时传播其净化后的 request id;request id 仍是不可信的诊断 metadata。 +DO invoke envelope 通过 canonical namespace、worker、version 和 storage id 标识 persisted bundle,不接受 inline worker source。 +Tenant-facing DO object name/id 必须是 well-formed Unicode string。`idFromName()` / `idFromString()` 会在 hash 或 dispatch 前拒绝 lone UTF-16 surrogate;do-runtime alarm ingress 和 Workflows revalidation 执行相同边界。 +DO host id 最多 512 UTF-8 bytes,并使用不带前导零的 canonical `shardN` suffix。DO binding class name 使用 ASCII JavaScript class-name grammar,并在 deploy 时限制为最多 468 bytes,确保所有 shard suffix 都能满足 aggregate host-id 上限。 ## Redis / Storage 合同 @@ -67,14 +72,17 @@ workerd 2026-07-01 会大小写不敏感地拒绝 SQLite reserved `_cf_` namespa - 同一时间只有一个 task 拥有一个 class shard。 - Generation fence 防止 stale owner 在 ownership 移动后继续 commit。 +- `do-runtime/protocol.js` 持有 DO ownership error vocabulary。Injected runtime transport 将 retry 和 stale-hint 子集保持为私有策略,并通过 response-classification test pin 住。 - Facet identity 是 stable `doStorageId` 内的 `className:objectName`,因此 worker promotion 保留 object state。 - 已构造的 native facet 会保留构造时的 class version,直到 host actor restart 或 facet deletion。Promotion 改变未来 load 和 routing metadata,不会替换当前 host actor 中已经构造的 facet。 - Whole-worker delete 后 redeploy 会分配新的 `doStorageId`;旧 native storage tombstone 给后续 cleanup,而不是立即物理删除。 - WebSocket upgrade 必须在 owner endpoint 上完成。Owner-hinted WebSocket direct retry 不能 fall back 到 router-established 101。 - WDL 会尽量让 client-facing WebSocket 由 gateway 持有并保持连接,包括 user-runtime 或 do-runtime restart 后的 backend reconnect。这个连接连续性目标强于 Cloudflare shutdown 行为;Cloudflare 可以终止 WebSocket 让新 Durable Object instance 接管。当前 backend facet 仍属于 owner scope:初始 `101` 之后,WebSocket message / close event 不会在每一帧重新校验 Redis owner generation。未来增强应在尽量保持 client 连接的同时 rebinding 或拒绝 stale backend owner facet,而不是把 client disconnect 作为主要安全机制。Gateway 重置 backend reconnect epoch 时,旧 epoch 下排队的 client message 可能被丢弃,且没有逐帧 ack/nack。 -- Ordinary fetch/RPC 只有在明确的 stale-owner 或 owner-race response 后才会回退到 router。Direct owner transport failure,或 direct owner hop 返回 502/503/504 且没有 fresh owner-hint header 时,会清除 cached hint。安全的 `GET`/`HEAD` request 可以通过 router 重放以重新发现 owner;非幂等 method 和 RPC 会返回 `owner_unavailable`,不会重放,因为 owner 可能已经应用了该请求。 +- Ordinary fetch/RPC 只有在明确的 stale-owner 或 owner-race response 同时携带 do-runtime 私有 ownership-error control header 时才会回退到 router;tenant response body 不能触发重放。Direct owner transport failure,或 direct owner hop 返回 502/503/504 且没有 fresh owner-hint header 时,会清除 cached hint。安全的 `GET`/`HEAD` request 可以通过 router 重放以重新发现 owner;非幂等 method 和 RPC 会返回 `owner_unavailable`,不会重放,因为 owner 可能已经应用了该请求。 +- Shared runtime transport 统一持有 host binding 与 injected facade 的 owner-hint cache wiring、invoke race retry 和 response-header stripping。Connect wrapper 刻意不包含 invoke-only router fallback,以保留 owner-established WebSocket upgrade 语义。 - `WEBSOCKET_RECONNECT_DELAYS_MS` 和 `WEBSOCKET_MAX_BUFFERED_MESSAGES` 可以在不改代码的情况下调整 gateway backend reconnect budget 和 client-message buffer cap。 - Alarm delivery 是 at-least-once。Scheduler 唤醒 Workflows;Workflows 把到期 internal alarm job promote 到 ready,在 DB 2 run token 下 claim,然后调用 do-runtime `/internal/do/alarms/dispatch`。do-runtime 仍然构造原来的 `DoInvoke{kind:"alarm"}` 请求,并走正常 owner router/fence 路径。 +- Alarm mutation、retarget、dispatch 和 whole-worker storage cleanup 只接受 canonical positive JavaScript-safe-integer worker version grammar。非法 internal 或 persisted version 会在写入 job 或尝试 worker invoke 前失败。 - Alarm due time 是传给 `setAlarm()` 的 Unix millisecond timestamp。Workflows 和 do-runtime 都用各自本地 wall clock 判断这些 timestamp;如果 backend ready hint 在 SQLite alarm row 对 do-runtime 来说尚未到期时抵达,do-runtime 会 ignore 这次 dispatch,但不清 row,让 backend due-index repair 路径之后继续投递。这是 alarm compatibility 边界,不属于 Redis-time owner lease fence。 - Failed alarm 使用 `WORKFLOWS_DO_ALARM_RETRY_DELAY_MS`、`WORKFLOWS_DO_ALARM_RETRY_MAX_DELAY_MS` 和 `WORKFLOWS_DO_ALARM_RETRY_JITTER` 的 exponential backoff 和 jitter 重试,最多到 `WORKFLOWS_DO_ALARM_RETRY_MAX_TRIES`(默认 `6`),之后 discard 并增加 `do_alarm_dispatches{outcome="discarded"}`。 - 如果 Workflows client 调用 do-runtime 后 timeout,backend 会保留 running claim 到 `WORKFLOWS_DO_ALARM_CLAIM_LEASE_MS` 过期,而不是立即调度 retry。默认值是五分钟,且配置值会被 clamp 到高于 `WORKFLOWS_DISPATCH_TIMEOUT_MS`,这样正常 timeout 处理可以避免 do-runtime 仍在执行原 dispatch 时并发执行重叠 alarm body。Operator 应按最长预期 alarm handler body 配置 claim lease,而不只是按 HTTP dispatch timeout;alarm body 仍是 at-least-once,claim lease 过期后可能重叠执行。 @@ -82,7 +90,7 @@ workerd 2026-07-01 会大小写不敏感地拒绝 SQLite reserved `_cf_` namespa Owner resolution 是单写入协议: 1. do-runtime 从 `doStorageId`、class name 和 shard 派生 owner scope。 -2. Claim 或 renew 前 WATCH owner record、generation key 和 active worker storage pointer。 +2. Owner resolution 会 WATCH owner record、generation key、worker delete lock 和 active worker storage pointer。`whole` delete lock 会拒绝 ownership;`version` lock 仍属于 WATCH snapshot,但不会中断 active storage。这个 WATCH 会阻止 claim 在 whole-worker delete 开始后提交。Renew 会单独 WATCH owner record 和 active storage pointer;generation fence 已包含在 owner record 中,不需要再次读取 generation key。 3. 如果另一个 task 持有 live owner,router 返回该 owner 或 owner-hint header;runtime facade 可以直连重试,但 owner task 仍会重新检查 fence。 4. 如果 owner 缺失或过期,claimant 在一个 Redis transaction 中递增 monotonic generation counter,并写入带 TTL 的 owner record。 5. Local dispatch 使用 native facet 前检查 `taskId`、`generation`、lease expiry、active `doStorageId` 和剩余 lease budget。stale generation、expired lease 或 storage pointer 改变都会 fail closed。剩余 lease 小于 `DO_OWNER_LEASE_GUARD_MS`(默认 `1000`)时,owner 会先尝试 same-task、same-generation CAS renew;如果 renew 失败,才 fail closed。这个 guard 缩窄 takeover window,但不是 per-SQL-call 或 SQLite commit-time fence。 @@ -97,8 +105,11 @@ Terraform 除了 Fargate task memory limit,还会给 do-runtime workerd contai - do-runtime internal endpoints 只在 private mesh 内可达,并要求共享的 `WDL_INTERNAL_AUTH_TOKEN` / `x-wdl-internal-auth` 内部认证 header。Health 和 metrics endpoint 例外。 - Tenant code 只能通过 runtime 生成的 facade 和 frozen metadata 访问 DO。 - Tenant-visible DO metadata 和 error 不得包含 owner task id、backend endpoint 或原始 transport error 文本。 -- Owner hint 只信任 do-runtime header,并且要通过 endpoint grammar validation。 -- Owner-hint 防御是分层的:忽略 tenant response body,只信任 do-runtime control header,并且必须通过 endpoint grammar / acceptable-address 检查。 +- Owner hint 只信任 do-runtime header,并且要通过 endpoint grammar validation。Owner hint 和 invoke fence 必须携带正的 JavaScript-safe-integer generation。 +- Task identity 和 persisted owner record 在写入和读取时都会校验。Persisted record 的 `ownerKey`、`hostId`、storage id、class 和 shard 必须能重建出读取它的 Redis scope;owner resolution 还必须在 do-runtime 读取 invoking bundle 的 active storage pointer 前,确认 record 的 canonical namespace 和 worker 与该 bundle 一致。Owner forwarding 只接受 8788 端口上的 DO service/headless DNS,或 RFC1918/100.64 私网 IPv4;非法记录在附加 internal auth 前 fail closed。 +- Owner-hint 与 ownership-error 防御是分层的:忽略 tenant response body 和 tenant-supplied control header,只信任 do-runtime control header;hint 还必须通过 endpoint grammar / acceptable-address 检查。 +- 注入的 DO transport 与共享 D1/DO endpoint validator 会在 tenant module 前执行,并捕获 private-header stripping、request bound、invoke serialization、replay classification 和 endpoint validation 使用的 intrinsic。Tenant prototype mutation 不能在校验后改写受信 target 或 replay policy。 +- 注入的 alarm shim 也会在 tenant module 前执行,并捕获 internal alarm 分类、SQLite 状态更新和 storage facade 安装依赖的 request、response、number、proxy 与 reflection 操作。Tenant 顶层对这些 intrinsic 的修改不能把 internal alarm 重定向到 tenant fetch handler,也不能阻止 facade 安装。 - do-runtime supervisor 必须调用本地 `127.0.0.1:8788` drain/renew endpoint;Service Connect alias 可能打到其他 task。 ## 可观测性 @@ -137,7 +148,7 @@ do-runtime 围绕 owner resolution、dispatch、alarm execution、drain、renew ## 已知约束和非目标 - 当前 lifecycle 不会在 worker delete 时物理清除 native facet SQLite storage。 -- Whole-worker delete 会删除 active `worker:do-storage::` pointer,并在 delete commit 后请求 Workflows 删除 internal DO alarm jobs;pointer 消失后,旧 facet 的 late `setAlarm()` 写入会被忽略。Cleanup 会 fence 到被删除的 `doStorageId`,因此同名 worker 以新 storage id redeploy 后不会被旧 delete 扫掉。如果 best-effort cleanup 失败,远期残留 alarm job 可能留在 DB 2 直到到期;随后 dispatch 会因为 storage pointer 已消失而自清理。`do:objects:` 会作为未来 platform cleanup 的 tombstone 保留。 +- Whole-worker delete 会删除 active `worker:do-storage::` pointer,并在 delete commit 后请求 Workflows 删除 internal DO alarm jobs;pointer 消失后,旧 facet 的 late `setAlarm()` 写入会被忽略。Cleanup 会 fence 到被删除的 `doStorageId`,因此同名 worker 以新 storage id redeploy 后不会被旧 delete 扫掉。如果 best-effort cleanup 失败,远期残留 alarm job 可能留在 DB 2 直到到期;随后 dispatch 会因为 storage pointer 已消失而自清理。First owner claim 会 WATCH 同一把 per-worker delete lock 和 storage pointer;只有 `whole` lock kind 会拒绝 ownership,因此删除 inactive version 不会中断 active worker,而 whole-worker delete 也不会漏掉最终 owner scan 之后创建的 owner/generation state。`do:objects:` 会作为未来 platform cleanup 的 tombstone 保留。 - DO object registry 写入是 best-effort。Registry 写失败时 dispatch 会继续,因此 tombstone set 可能不完整;未来 cleanup 必须容忍缺失 member,并把 active storage pointer、owner/alarm state 当成更强的 lifecycle signal。 - Gateway-held WebSocket recovery 只对 client connection continuity 做 best-effort。Backend DO facet 在初始 `101` 之后不会逐消息 re-fence;owner handoff 安全依赖 reconnect/rebind 行为,以及创建 backend facet 前运行的 owner-side dispatch fence。Gateway 重置 backend reconnect epoch 时,旧 epoch 下排队的 client message 可能被丢弃,且没有逐帧 ack/nack。 - Owner-hinted WebSocket direct retry 失败不会 fall back 到 router,因为最终 101 必须来自 owner endpoint。 diff --git a/docs/modules/gateway.md b/docs/modules/gateway.md index 983d976..a22b02b 100644 --- a/docs/modules/gateway.md +++ b/docs/modules/gateway.md @@ -28,11 +28,16 @@ Gateway has three dispatch branches: The resolved `{ ns, worker, version }` becomes `x-worker-id: ::` and `x-worker-prefix` on the runtime request. Literal `__system__` routes go to `RUNTIME_SYSTEM`; all ordinary tenant namespaces go to `RUNTIME_USER`. +Before forwarding, gateway removes client-supplied `x-worker-id`, `x-worker-prefix`, +and every `x-wdl-*` header. The same internal-header policy filters WebSocket upgrade +responses before they cross back onto the public socket. The `ADMIN_HOST` branch is infrastructure traffic, not a loaded-worker request. It does not set `x-worker-id` or `x-worker-prefix`. `PLATFORM_DOMAIN` and -`ADMIN_HOST` are environment-configurable; the code defaults are `workers.local` -and unset admin-host short-circuiting. +`ADMIN_HOST` are environment-configurable. All tiers normalize `PLATFORM_DOMAIN` as an +ALB-compatible ASCII DNS hostname of at most 126 bytes, with an alphabetic final label, +and default it to `workers.local`; one trailing dot is removed and the result is +lowercased. Admin-host short-circuiting remains unset by default. ## Interfaces diff --git a/docs/modules/gateway.zh.md b/docs/modules/gateway.zh.md index 096bf51..f4e20e2 100644 --- a/docs/modules/gateway.zh.md +++ b/docs/modules/gateway.zh.md @@ -16,7 +16,9 @@ Gateway 有三条 dispatch 分支: 解析出的 `{ ns, worker, version }` 会转成 runtime 请求头 `x-worker-id: ::` 和 `x-worker-prefix`。字面量 `__system__` route 进入 `RUNTIME_SYSTEM`;普通 tenant namespace 进入 `RUNTIME_USER`。 -`ADMIN_HOST` 分支是 infrastructure traffic,不是 loaded-worker request。它不设置 `x-worker-id` 或 `x-worker-prefix`。`PLATFORM_DOMAIN` 和 `ADMIN_HOST` 可通过环境变量配置;代码默认是 `workers.local`,且未设置 admin-host short-circuit。 +转发前,gateway 会清除客户端提供的 `x-worker-id`、`x-worker-prefix` 和所有 `x-wdl-*` header。同一套内部 header 策略也会在 WebSocket upgrade response 回到公开 socket 前进行过滤。 + +`ADMIN_HOST` 分支是 infrastructure traffic,不是 loaded-worker request。它不设置 `x-worker-id` 或 `x-worker-prefix`。`PLATFORM_DOMAIN` 和 `ADMIN_HOST` 可通过环境变量配置。所有 tier 都会把 `PLATFORM_DOMAIN` normalize 成最多 126 bytes、final label 仅包含字母的 ALB-compatible ASCII DNS hostname;单个尾点会被移除,结果转成小写,并默认使用 `workers.local`。Admin-host short-circuit 默认仍未设置。 ## 接口 diff --git a/docs/modules/infra.md b/docs/modules/infra.md index 1f3fa4d..708da28 100644 --- a/docs/modules/infra.md +++ b/docs/modules/infra.md @@ -99,13 +99,14 @@ Private mesh callers and receivers share `WDL_INTERNAL_AUTH_TOKEN` as the curren internal token. Calls between runtime, d1-runtime, do-runtime, scheduler, workflows, and redis-proxy sidecars carry it as `x-wdl-internal-auth`. Receivers also accept optional `WDL_INTERNAL_AUTH_PREVIOUS_TOKEN` during rotation; callers -always send only the current token. Both token values must be non-empty ASCII -strings so JS and Rust compare the same header representation. Health and -metrics endpoints are the only unauthenticated service endpoints. The token is -platform plumbing, not a tenant binding: runtime wrapper code strips it from -tenant-visible `env`, host-owned DO proxies and host-side backend capabilities -add it for DO forwarding, and spoofed tenant headers are removed before -forwarding. +always send only the current token. Receivers require exactly one auth header. +Both token values must contain only visible ASCII bytes, with no whitespace or commas, +so Fetch header normalization preserves them exactly and header joining cannot make +repeated headers look like one configured token. Health and metrics endpoints are the +only unauthenticated service endpoints. The token is platform plumbing, +not a tenant binding: runtime wrapper code strips it from tenant-visible `env`, host-owned +DO proxies and host-side backend capabilities add it for DO forwarding, and spoofed tenant +headers are removed before forwarding. ## Redis / Storage Contracts @@ -129,6 +130,11 @@ Stateful storage: multi-replica safe, but rollout can still pause scheduling because ECS uses stop-before-start replacement. - Workflows is a separate Rust service. +- By default, Scheduler and Workflows drain in-flight work for up to 25 seconds. + Compose, Kubernetes, and Terraform ECS pin 30-second stop windows so the platform + does not terminate either process before its default application drain closes. + Deployments that override `SCHEDULER_SHUTDOWN_DRAIN_MS` or + `WORKFLOWS_SHUTDOWN_DRAIN_MS` should review the corresponding stop window. - Gateway, user-runtime, and system-runtime can be horizontally replicated behind the environment's load balancing and service discovery layer. Local route caches, loaded isolates, and owner hints are optimizations, not authority. @@ -158,6 +164,9 @@ Stateful storage: - Runtime internal `:8088`, d1-runtime `:8787`, do-runtime `:8788`, workflows `:9120`, and Redis are private mesh services. Private service calls also require `x-wdl-internal-auth` with the shared `WDL_INTERNAL_AUTH_TOKEN`. +- Kubernetes NetworkPolicies use per-component caller sets. ECS intentionally groups + user-runtime, system-runtime, D1, and DO tasks in one runtime security group, so its + network rules are coarser and are not required to match the Kubernetes caller matrix. - Tenant-running runtime tasks use least-privilege ECS task roles, public-only workerd outbound bindings, and private mesh security groups as their cloud credential and network boundary. ECS Exec should be enabled only where operator access is intended. diff --git a/docs/modules/infra.zh.md b/docs/modules/infra.zh.md index df452c8..5a961c7 100644 --- a/docs/modules/infra.zh.md +++ b/docs/modules/infra.zh.md @@ -35,8 +35,7 @@ Local compose 是开发便利环境,不是 production delivery contract。它 ## 接口 -- 公开入口通过 ALB/gateway。Terraform-managed ALB ingress 可以在同一个 gateway - service 上承载 admin host、platform wildcard、canonical site host 和额外 exact public host。 +- 公开入口通过 ALB/gateway。Terraform-managed ALB ingress 可以在同一个 gateway service 上承载 admin host、platform wildcard、canonical site host 和额外 exact public host。 - Admin-host ingress 通过 gateway 的 `ADMIN_HOST` 分支进入 system-runtime control。 - Local Compose private service hop 使用 `envoy/envoy.yaml` 和编译进 `dist/workerd-configs` 的 `*-local.capnp` workerd config。 - ECS Service Connect 覆盖 runtime、D1、DO 和 workflows service target。 @@ -59,7 +58,7 @@ Gateway 和 runtime 在不同环境中使用稳定的内部端口合同: 不要把 private mesh endpoint 暴露到 public ingress。K8s 和 ECS 交付都必须通过 Service Connect、ClusterIP Services、NetworkPolicy 或等价控制保持这个假设成立。 -Private mesh caller 和 receiver 共享 `WDL_INTERNAL_AUTH_TOKEN` 作为当前 internal token。runtime、d1-runtime、do-runtime、scheduler、workflows 和 redis-proxy sidecar 之间的调用会携带 `x-wdl-internal-auth`。轮换期间 receiver 还会接受可选的 `WDL_INTERNAL_AUTH_PREVIOUS_TOKEN`;caller 始终只发送当前 token。两个 token 值都必须是非空 ASCII string,确保 JS 和 Rust 比较同一种 header 表示。Health 和 metrics endpoint 是唯一不要求该 token 的 service endpoint。这个 token 是平台 plumbing,不是 tenant binding:runtime wrapper code 会从 tenant-visible `env` 中剥离它,host-owned DO proxy 和 host-side backend capability 会在 DO forwarding 时添加它,并在 forwarding 前删除租户伪造的同名 header。 +Private mesh caller 和 receiver 共享 `WDL_INTERNAL_AUTH_TOKEN` 作为当前 internal token。runtime、d1-runtime、do-runtime、scheduler、workflows 和 redis-proxy sidecar 之间的调用会携带 `x-wdl-internal-auth`。轮换期间 receiver 还会接受可选的 `WDL_INTERNAL_AUTH_PREVIOUS_TOKEN`;caller 始终只发送当前 token。Receiver 要求 auth header 恰好出现一次;两个 token 值都只能包含 visible ASCII byte,不能包含空白或逗号,确保 Fetch header normalization 不改变 token,并避免 header joining 把重复 header 伪装成单个已配置 token。Health 和 metrics endpoint 是唯一不要求该 token 的 service endpoint。这个 token 是平台 plumbing,不是 tenant binding:runtime wrapper code 会从 tenant-visible `env` 中剥离它,host-owned DO proxy 和 host-side backend capability 会在 DO forwarding 时添加它,并在 forwarding 前删除租户伪造的同名 header。 ## Redis / Storage 合同 @@ -80,6 +79,7 @@ Stateful storage: - Scheduler 部署默认 1 个副本;当前 dispatch 路径具备多副本安全性,但 ECS rollout 仍使用 stop-before-start replacement,部署期间可能短暂停止调度。 - Workflows 是独立 Rust service。 +- Scheduler 和 Workflows 默认最多 drain 25 秒 in-flight work;Compose、Kubernetes 和 Terraform ECS 均固定 30 秒 stop window,避免平台在默认应用 drain 结束前终止进程。部署如覆盖 `SCHEDULER_SHUTDOWN_DRAIN_MS` 或 `WORKFLOWS_SHUTDOWN_DRAIN_MS`,应同步评估对应的 stop window。 - Gateway、user-runtime 和 system-runtime 可以放在环境的 load balancing / service discovery 层后面水平扩展。本地 route cache、已加载 isolate 和 owner hint 都是优化,不是 authority。 - D1/DO 使用 owner lease、monotonic generation fence 和本地 drain/renew。超过 1 个 task 时需要稳定的 per-replica storage identity,并确保 supervisor drain/renew 只走私有本地入口,不能通过 service alias 打到其他 replica。 - 普通 D1/DO task 丢失会退回到 lease expiry,再由其他 replica takeover;graceful rollout 应优先走 supervisor drain,在 child workerd process 退出前释放 ownership。 @@ -92,6 +92,7 @@ Stateful storage: - user-runtime loaded worker outbound 是 public-only。 - system-runtime 是刻意放宽的特权环境。 - Runtime internal `:8088`、d1-runtime `:8787`、do-runtime `:8788`、workflows `:9120` 和 Redis 都是 private mesh services。Private service call 还必须携带带有共享 `WDL_INTERNAL_AUTH_TOKEN` 的 `x-wdl-internal-auth`。 +- Kubernetes NetworkPolicy 使用按组件区分的 caller 集合。ECS 刻意让 user-runtime、system-runtime、D1 和 DO task 共用一组 runtime security group,因此其网络规则粒度更粗,不要求与 Kubernetes caller 矩阵相同。 - Tenant-running runtime task 使用 least-privilege ECS task role、public-only workerd outbound binding 和 private mesh security group 作为 cloud credential 与 network 边界。ECS Exec 只应在需要 operator access 的地方启用。 ## 可观测性 diff --git a/docs/modules/log-tail-observability.md b/docs/modules/log-tail-observability.md index 86484e1..e60b3ef 100644 --- a/docs/modules/log-tail-observability.md +++ b/docs/modules/log-tail-observability.md @@ -11,6 +11,9 @@ Shared JS primitives live in `shared/observability.js`. Runtime tailing uses: - `runtime/tail-worker.js` for console/exception capture. - `runtime/tail-forwarder.js` for active-set checks and append POSTs. +- The same runtime forwarder owns invocation start/finish envelope timing and + thrown-value normalization, so gateway dispatch and service-binding fetches emit one + `worker_fetch` shape. - `redis-proxy` logs endpoints for active tail checks and stream appends. - `control/handlers/logs-tail.js` for SSE sessions and heartbeat activation. - CLI `wdl tail` for user consumption. @@ -134,13 +137,22 @@ Central owners: Common rules: -- `x-request-id` propagates across gateway, control/runtime, loaded workers, and D1 - where possible. Missing inbound ids are minted at ingress; dirty ids such as - multi-valued, control-character, or overly long ids are treated as absent. JS - entrypoints using `shared/request-scope.js` echo the sanitized id on responses and - log it as `request_id` on `request_complete`; Rust sidecars sanitize inbound ids and - log them where request middleware owns completion. Control's Redis `PUBLISH` path logs - the id locally but does not put it in the pub/sub payload. +- `x-request-id` propagates across gateway, control/runtime, loaded workers, D1, and Workflows + where possible. Missing inbound ids are minted at ingress. Header adapters consider + only the first comma-delimited token; ids containing bytes outside visible ASCII, + quotes, backslashes, or more than 128 bytes are treated as absent. JS entrypoints using + `shared/request-scope.js` echo the sanitized id on responses and log it as + `request_id` on `request_complete`; Rust sidecars sanitize inbound ids and log them + where request middleware owns completion. Control's Redis `PUBLISH` path logs the id + locally but does not put it in the pub/sub payload. +- Loaded-worker host facades keep request ids in invocation-local ALS across native + Promise and `async` continuations. Private do-runtime fetch, alarm, and RPC dispatches + carry the id in `x-request-id`. Custom thenables are not a supported propagation + boundary; the wrapper does not inspect or replace tenant return values to extend it. + Ordinary nested handler calls without a new id inherit the current ALS context. + Tenant code can deliberately switch the isolate's ambient ALS frame, so this + propagation is best-effort diagnostic correlation rather than an authorization or + integrity boundary. - Logs use snake_case fields. Only `level=error` is emitted on stderr; debug/info/warn JSON log lines go to stdout so log routing stays identical across JS, Rust, and embedded workerd shims. @@ -149,8 +161,9 @@ Common rules: `ts`, `service`, `level`, `event`, plus snake_case fields. JS services use `shared/observability.js`; Rust services use `wdl-rust-common::log::emit_log_line`. Error text is emitted as `error_message`; - secret values, raw credentials, token material, raw Redis keys, and unbounded payloads - are not emitted. + throwables that cannot be converted to text degrade to `Unknown error` instead of + replacing the original failure. Secret values, raw credentials, token material, raw + Redis keys, and unbounded payloads are not emitted. - Product API response bodies should use camelCase unless an endpoint explicitly documents a different wire contract. - Metrics should use bounded enumerated labels only. @@ -160,10 +173,11 @@ Common rules: `request_complete` logs as `error_code` / `error_message`. - JS and Rust observability implementations intentionally share metric prefix `wdl`, request metric families `requests` / `request_duration_ms` / `request_errors`, - cardinality warning threshold `100`, and Prometheus content type - `text/plain; version=0.0.4; charset=utf-8`. The shared fixture - `tests/fixtures/observability-contract.json` pins those values without introducing - a runtime metrics owner. + cardinality warning threshold `100`, Prometheus content type + `text/plain; version=0.0.4; charset=utf-8`, and the structured-log key order, level + priorities, stream routing, and timestamp shape. The shared fixture + `tests/fixtures/observability-contract.json` pins those values without introducing a + runtime observability owner. - redis-proxy records KV payload sizes in the `kv_value_bytes` summary with only bounded `service`/`operation`/`kind` labels. It records value, metadata, and raw batch byte counts so operators can decide whether large-value offload is needed; diff --git a/docs/modules/log-tail-observability.zh.md b/docs/modules/log-tail-observability.zh.md index 1892098..2976de9 100644 --- a/docs/modules/log-tail-observability.zh.md +++ b/docs/modules/log-tail-observability.zh.md @@ -10,6 +10,7 @@ Observability 提供有界 metrics、结构化日志、request-id 传播和 live - `runtime/tail-worker.js` 捕获 console/exception。 - `runtime/tail-forwarder.js` 做 active-set check 和 append POST。 +- 同一个 runtime forwarder 统一持有 invocation start/finish envelope timing 和 thrown-value normalization,因此 gateway dispatch 与 service-binding fetch 会输出同一种 `worker_fetch` shape。 - `redis-proxy` logs endpoints 做 active tail check 和 stream append。 - `control/handlers/logs-tail.js` 管理 SSE session 和 heartbeat activation。 - CLI `wdl tail` 给用户消费。 @@ -90,13 +91,14 @@ WDL 在 JS workerd tier 和 Rust 服务中使用同一套可观测性策略: 通用规则: -- `x-request-id` 尽可能跨 gateway、control/runtime、loaded worker 和 D1 传播。缺失的 inbound id 会在入口生成;multi-valued、包含 control char 或过长的脏 id 会被当成缺失。使用 `shared/request-scope.js` 的 JS entrypoint 会在 response 中 echo sanitize 后的 id,并在 `request_complete` 日志中记录 `request_id`;Rust sidecar 会 sanitize inbound id,并在 request middleware 拥有 completion 时记录。Control 的 Redis `PUBLISH` 路径只在本地日志记录该 id,不把它放进 pub/sub payload。 +- `x-request-id` 尽可能跨 gateway、control/runtime、loaded worker、D1 和 Workflows 传播。缺失的 inbound id 会在入口生成;header adapter 只考虑第一个逗号分隔 token,包含 visible ASCII 以外字节、引号、反斜杠或超过 128 字节的 id 会被当成缺失。使用 `shared/request-scope.js` 的 JS entrypoint 会在 response 中 echo sanitize 后的 id,并在 `request_complete` 日志中记录 `request_id`;Rust sidecar 会 sanitize inbound id,并在 request middleware 拥有 completion 时记录。Control 的 Redis `PUBLISH` 路径只在本地日志记录该 id,不把它放进 pub/sub payload。 +- Loaded-worker host facade 通过 invocation-local ALS 在 native Promise 和 `async` continuation 中保留 request id;do-runtime 的私有 fetch、alarm 和 RPC dispatch 会在 `x-request-id` 中携带该 id。Custom thenable 不属于受支持的传播边界;wrapper 不会为延长 context 而检查或替换 tenant 返回值。普通的无新 id 嵌套 handler 调用会继承当前 ALS context;tenant code 也可以主动切换 isolate 的 ambient ALS frame,因此这只是 best-effort 的诊断关联,不是授权或完整性边界。 - 日志字段使用 snake_case。只有 `level=error` 写入 stderr;debug/info/warn JSON log line 都写入 stdout,这样 JS、Rust 和 embedded workerd shim 的日志路由保持一致。 -- 内部运维日志,包括 system-worker cleanup 日志和防御性的 Redis callback warning,也使用同一套单行 JSON envelope:`ts`、`service`、`level`、`event`,再加 snake_case 字段。JS 服务使用 `shared/observability.js`;Rust 服务使用 `wdl-rust-common::log::emit_log_line`。错误文本写入 `error_message`;不得输出 secret 值、raw credential、token material、raw Redis key 或无界 payload。 +- 内部运维日志,包括 system-worker cleanup 日志和防御性的 Redis callback warning,也使用同一套单行 JSON envelope:`ts`、`service`、`level`、`event`,再加 snake_case 字段。JS 服务使用 `shared/observability.js`;Rust 服务使用 `wdl-rust-common::log::emit_log_line`。错误文本写入 `error_message`;无法转成文本的 throwable 降级为 `Unknown error`,不能反向覆盖原始失败。不得输出 secret 值、raw credential、token material、raw Redis key 或无界 payload。 - 产品 API response body 默认使用 camelCase,除非 endpoint 明确记录不同 wire contract。 - Metrics 只使用有界枚举 label。 - 暴露 request metrics 的 Rust HTTP sidecar 使用同一组 `requests`、`request_duration_ms` 和 `request_errors` metric family,以及有界的 `service`/`route`/`status` labels;per-route error context 只进入 `request_complete` 日志中的 `error_code` / `error_message`。 -- JS 和 Rust observability 实现刻意共享 metric prefix `wdl`、request metric families `requests` / `request_duration_ms` / `request_errors`、cardinality warning threshold `100`、Prometheus content type `text/plain; version=0.0.4; charset=utf-8`;共享 fixture `tests/fixtures/observability-contract.json` pin 住这些值,但不引入 runtime metrics owner。 +- JS 和 Rust observability 实现刻意共享 metric prefix `wdl`、request metric families `requests` / `request_duration_ms` / `request_errors`、cardinality warning threshold `100`、Prometheus content type `text/plain; version=0.0.4; charset=utf-8`,以及结构化日志的 key 顺序、level priority、stream routing 和 timestamp shape;共享 fixture `tests/fixtures/observability-contract.json` pin 住这些值,但不引入 runtime observability owner。 - redis-proxy 用 `kv_value_bytes` summary 记录 KV payload size,label 只有有界的 `service`/`operation`/`kind`。它记录 value、metadata 和 raw batch byte count,用来判断是否需要 large-value offload;namespace、key 和 object identity 不进入 metric label。 - Rust `request_complete` log 输出整数 `duration_ms`,让各服务日志字段保持稳定;Prometheus duration summary 仍保留浮点值。 - JS `MetricsRegistry` 在单个 metric name 达到 100 个 series 时输出一次结构化 `metric_cardinality_warning` 日志,之后会丢弃该 metric 的全新 series,但继续更新已有 series。Rust `MetricStore` 当前仍保持同一 warning-only tripwire。该 warning 携带 metric name、观测到的 series 数和配置 limit;tenant-specific 细节本来就不应进入 label。因为这条 warning 由 metrics registry 输出,所以不会被 `LOG_LEVEL` 抑制。 diff --git a/docs/modules/queues-cron.md b/docs/modules/queues-cron.md index cdeb70e..8198901 100644 --- a/docs/modules/queues-cron.md +++ b/docs/modules/queues-cron.md @@ -36,6 +36,13 @@ Scheduler owns runtime delivery: - Cron code lives under `rust/scheduler/src/cron/`. - Queue registry, consume, delayed delivery, DLQ, and orphan handling live under `rust/scheduler/src/queue/`. +- Before constructing a cron runtime worker id, scheduler revalidates the projection's + route namespace, worker name, and version with the canonical `wdl-rust-common` + grammar. Invalid cron refs are removed before slot advance or runtime dispatch, and + cron sweep does not recreate refs from malformed worker metadata. Queue consumer + dispatch identities are also revalidated before use. A noncanonical identity makes + the projection unusable and follows absent-consumer handling, so queued backlog may + move to the orphan stream. - Scheduler defaults to one replica in deployment, and current dispatch paths are multi-replica safe. Extra replicas improve runtime concurrency but do not imply zero-gap deployment: production rollout may still use stop-before-start semantics and @@ -91,7 +98,9 @@ Cron uses wall-clock minute slots: computes each entry's next fire time with croner and the configured timezone, rounds it to the minute slot, and writes refs into slot buckets. This is repair logic, not the authority. Control uses JavaScript `croner` only for the initial promote-time - slot placement; scheduler uses Rust `croner` for repair and advancement. + slot placement; scheduler uses Rust `croner` for repair and advancement. Worker + hashes without canonical identity or metadata are skipped instead of reseeding + invalid refs, and emit a bounded `invalid_identity` or `invalid_meta` reason. 3. Cron refs carry the entry generation. At fire time scheduler re-reads `crons::` and compares `gen`; missing metadata, corrupt JSON, or generation mismatch makes the ref stale and removes it from the slot. @@ -135,7 +144,8 @@ Queue dispatch is stream-driven rather than wall-clock driven: failures that cannot become valid by retrying the same bytes, such as queue-message decode failures or invalid queue dispatch bodies, go directly to DLQ without consuming the retry budget. Aggregate request-body-too-large responses are split - and retried with smaller batches first. + and retried with smaller batches first. Scheduler bounds each runtime response body + to 1 MiB before parsing it. 5. The delayed loop wakes from `queue-delayed-wake` and from wall-clock sleeps until the next due delayed member. Each due member first takes a `queue-delayed-claim:*` lease sized to `SCHEDULER_FIRE_TIMEOUT_MS + 5000ms`; the winner moves it back to the main @@ -177,6 +187,9 @@ queue-orphaned:: Stream, messages whose consumer disappeared queue-delayed-wake Stream, wake signal for delayed ZSET dispatcher ``` +`wdl-rust-common::queue_keys` owns the wake stream and its `delayed_key` / `visible_at` +entry fields so the redis-proxy producer and scheduler consumer cannot drift. + Indexes are not authority. Writers add index entries, and scheduler reconcile owns stale cleanup after proving the referenced key is absent. @@ -257,8 +270,10 @@ Important cron signals: - `cron_queue_lag_ms{outcome=...}` - `cron_bucket_size` - `cron_stale_refs_cleaned` +- `cron_sweep_entries_skipped` +- `cron_sweep_workers_skipped` - Logs: `cron_fired`, `cron_lease_lost`, `cron_ref_stale`, `cron_ref_stale_advanced`, - `cron_reconcile` + `cron_sweep_entry_skipped`, `cron_sweep_worker_skipped`, `cron_reconcile` Important queue signals: @@ -294,6 +309,8 @@ Representative test anchors: consumers. - `tests/unit/control-lifecycle-indexes.test.js`: JS cron/queue key helpers and projection staging. +- `tests/fixtures/queue-key-parse.json`: shared JS/Rust queue discovery-key parser + contract. - `tests/unit/runtime-lib.test.js`: internal dispatch body normalization. - `tests/unit/runtime-dispatch-handlers.test.js`: scheduled and queue dispatch behavior, tail event envelope behavior. diff --git a/docs/modules/queues-cron.zh.md b/docs/modules/queues-cron.zh.md index 00576a0..0962a04 100644 --- a/docs/modules/queues-cron.zh.md +++ b/docs/modules/queues-cron.zh.md @@ -23,6 +23,7 @@ Scheduler 负责实际投递: - Cron 代码在 `rust/scheduler/src/cron/`。 - Queue registry、consume、delayed delivery、DLQ 和 orphan 处理在 `rust/scheduler/src/queue/`。 +- Scheduler 在构造 cron runtime worker id 前会用 canonical `wdl-rust-common` grammar 重新校验 projection 的 route namespace、worker name 和 version。非法 cron ref 会在 slot advance 或 runtime dispatch 前移除,cron sweep 也不会从 malformed worker metadata 重新播种 ref。Queue consumer 的 dispatch identity 也会在使用前重新校验;noncanonical identity 会使 projection 不可用,并按 consumer absent 处理,因此 backlog 可能进入 orphan stream。 - 部署上 scheduler 默认 1 个副本,当前 dispatch 路径具备多副本安全性。增加副本可以提高运行时并发,但不等于部署零中断:生产 rollout 仍可采用 stop-before-start 语义,并短暂暂停调度。 ## 接口 @@ -60,7 +61,7 @@ Scheduler 不是通用 job runner。它把 Redis projection 转成 runtime call Cron 使用 wall-clock 分钟 slot: 1. `wait_ms_until_next_slot()` 睡到下一个 UTC 分钟边界。tick loop 随后扫描当前 `cron-slot:` bucket 和前一个 bucket。扫描前一个 bucket 是为了覆盖分钟 rollover 附近刚写入的 ref。 -2. 另一条 sweep/reconcile 路径从 `cron:index:workers` 读取 active cron hash,用 croner 和配置的 timezone 计算每个 entry 的下一次触发时间,再 round 到分钟 slot,并把 ref 写入 slot bucket。这个流程是 repair 逻辑,不是权威状态。Control 只在 promote-time 初始 slot placement 使用 JavaScript `croner`;scheduler repair 和 advance 使用 Rust `croner`。 +2. 另一条 sweep/reconcile 路径从 `cron:index:workers` 读取 active cron hash,用 croner 和配置的 timezone 计算每个 entry 的下一次触发时间,再 round 到分钟 slot,并把 ref 写入 slot bucket。这个流程是 repair 逻辑,不是权威状态。Control 只在 promote-time 初始 slot placement 使用 JavaScript `croner`;scheduler repair 和 advance 使用 Rust `croner`。缺少 canonical identity 或 metadata 的 worker hash 会被跳过,不会重新播种非法 ref,并输出 bounded `invalid_identity` 或 `invalid_meta` reason。 3. Cron ref 带 entry generation。真正 fire 前,scheduler 会重新读取 `crons::` 并比较 `gen`;metadata 缺失、JSON 损坏或 generation 不匹配都会让 ref 变成 stale,并从 slot 中移除。 4. Scheduler 会先原子地 lease ref、从当前 slot 移除、加入下一个 slot,然后才调用 runtime。这个顺序保证每个 slot 只 fire 一次,并且 runtime/network 失败不会变成自动 cron retry。 5. 如果某个 ref 滞留在早于当前 wall-clock slot 的旧 slot 中,scheduler 只把它 advance 到下一个 future slot,不会 fire。也就是说 outage 或 scheduler 长时间停顿期间错过的 cron event 会被跳过,而不是补发。 @@ -73,7 +74,7 @@ Queue dispatch 是 stream-driven,不是 wall-clock driven: 1. Producer 把 message envelope 写入 DB 1 stream;当 `delivery_delay` 或 retry delay 非零时,先写入 delayed ZSET。 2. Scheduler reconcile `queue:index:*` discovery set,为 live stream 创建固定的 `wdl-scheduler` consumer group,并维护一个内存中的 known delayed queue 集合。空 queue index 可以从权威 hash、stream 和 delayed ZSET 一次性 backfill;之后 projection 由 writer 维护,stale-index cleanup 由 reconcile 负责。 3. Consume loop 使用 `XREADGROUP` 读取 main stream,并按 `max_batch_size` 组 batch 投递,且 hard cap 为 `100`。每次 read 会用当前 active stream set 的 consumer batch-size snapshot 限制 `COUNT`,避免一次 poll 把超过当前 consumer 单批 dispatch 能力的 entries 放进 PEL。PEL reap 在 consumer 仍存在时使用同一 per-consumer cap;consumer 已消失的 orphan movement 仍可按 hard cap 分页。Consume 和 PEL reap 可以在 queue semaphore 下按 stream 并行 dispatch。每条 dispatch path 在向 runtime 发送 message 前都会重读该 stream 的权威 `queue-consumer` hash 并刷新内存 registry,因此 consumer promote 后,一旦 message 被选中,就不需要等下一次 reconcile tick 才使用新版本。当前模型里 `max_batch_timeout_ms` 不是 batching wait window。 -4. Runtime 返回 queue outcome envelope。Scheduler 解析 explicit `ack`、explicit `retry`、batch retry 和 implicit ack,然后用 Redis pipeline 执行 `XACK`/`XDEL`、delayed retry `ZADD`、DLQ append 或 immediate retry write。不会因为重试同一批 bytes 变好的平台失败,例如 queue message decode failure 或非法 queue dispatch body,会直接进 DLQ,不消耗 retry budget。聚合 request-body-too-large 会先拆成更小 batch 重试。 +4. Runtime 返回 queue outcome envelope。Scheduler 解析 explicit `ack`、explicit `retry`、batch retry 和 implicit ack,然后用 Redis pipeline 执行 `XACK`/`XDEL`、delayed retry `ZADD`、DLQ append 或 immediate retry write。不会因为重试同一批 bytes 变好的平台失败,例如 queue message decode failure 或非法 queue dispatch body,会直接进 DLQ,不消耗 retry budget。聚合 request-body-too-large 会先拆成更小 batch 重试;scheduler 在解析前把每个 runtime response body 限制为 1 MiB。 5. Delayed loop 由 `queue-delayed-wake` 和下一条 due member 的 wall-clock sleep 唤醒。每个到期 member 会先取得 `queue-delayed-claim:*` lease,TTL 为 `SCHEDULER_FIRE_TIMEOUT_MS + 5000ms`;抢到的副本把它移回 main stream,或在 consumer 已消失时移入 orphan stream。 6. Orphan / Pending-Entry cleanup 是诊断和保护机制。它防止 consumer 删除或 scheduler crash 路径静默丢消息,但 main queue stream 仍是 durable backlog,并且故意不 trim。Delayed ZSET 和 orphan stream-tail migration 受 `QUEUE_SWEEP_BATCH_SIZE` 分页控制,默认 `100`。 @@ -109,6 +110,8 @@ queue-orphaned:: Stream, consumer 消失后的 message 降落 queue-delayed-wake Stream, delayed ZSET dispatcher 的 wake signal ``` +`wdl-rust-common::queue_keys` 持有 wake stream 及其 `delayed_key` / `visible_at` entry fields,避免 redis-proxy producer 与 scheduler consumer 漂移。 + Index 不是权威状态。Writer 只负责添加 index member;scheduler reconcile 在确认被引用 key 不存在后负责 stale cleanup。 ## Ownership / 并发 / 失败语义 @@ -158,7 +161,9 @@ Scheduler 为 cron 和 queue outcome 输出结构化日志和 Prometheus metrics - `cron_queue_lag_ms{outcome=...}` - `cron_bucket_size` - `cron_stale_refs_cleaned` -- Logs:`cron_fired`、`cron_lease_lost`、`cron_ref_stale`、`cron_ref_stale_advanced`、`cron_reconcile` +- `cron_sweep_entries_skipped` +- `cron_sweep_workers_skipped` +- Logs:`cron_fired`、`cron_lease_lost`、`cron_ref_stale`、`cron_ref_stale_advanced`、`cron_sweep_entry_skipped`、`cron_sweep_worker_skipped`、`cron_reconcile` 重要 queue 信号: @@ -185,6 +190,7 @@ Runtime tail logs 也会为 loaded worker 执行输出 `worker_scheduled` 和 `w - `tests/unit/control-lib.test.js`:cron 和 queue manifest 解析。 - `tests/unit/control-routing.test.js`:cron 和 queue consumer 的 promote projection。 - `tests/unit/control-lifecycle-indexes.test.js`:JS cron/queue key helper 和 projection staging。 +- `tests/fixtures/queue-key-parse.json`:JS/Rust 共用的 queue discovery-key parser 合同。 - `tests/unit/runtime-lib.test.js`:internal dispatch body normalization。 - `tests/unit/runtime-dispatch-handlers.test.js`:scheduled / queue dispatch 行为,以及 tail event envelope 行为。 - `tests/unit/style-contracts.test.js`:跨 tier Redis key/layout drift 检查。 diff --git a/docs/modules/runtime.md b/docs/modules/runtime.md index 43ee18d..92c921e 100644 --- a/docs/modules/runtime.md +++ b/docs/modules/runtime.md @@ -21,6 +21,9 @@ are: Workers are loaded by immutable id `::`. Promotion creates a new id, so active-version changes naturally cold-load a fresh isolate. +`runtime/load.js` also owns the shared `workerLoader.get()` wrapper: every cache miss is +recorded for later eviction, while only active-version dispatch paths request sibling +eviction. Service bindings load their pinned version without evicting siblings. The two runtime pools use the same image and source but different capnp configs: @@ -84,9 +87,10 @@ services. Runtime therefore treats bindings as adapters: worker secrets. A worker-level secret with the same name wins over a namespace-level secret, which wins over a var. Control enforces a headroomed estimate of workerd's `workerLoader` serialized env budget during deploys and secret mutations. That - estimate includes user vars/secrets plus runtime-injected binding/workflow env values, - including required caller secret copies in platform/service binding props, so an - over-large env fails in the control plane instead of during runtime cold-load. + estimate calls the same `buildWorkerEnv()` materializer as cold-load with shape-only + factories, then measures user vars/secrets plus runtime-injected binding/workflow env + values, including required caller secret copies in platform/service binding props, so + an over-large env fails in the control plane instead of during runtime cold-load. - Stateful bindings such as D1, Durable Objects, and Workflows call dedicated backend services. The hidden backend Fetchers stay in runtime and are removed before tenant code observes `env`. @@ -128,7 +132,9 @@ are isolated by prefix. Runtime supports the common `head`, `get`, `put`, `delet 25 MiB cap. `put(stream, ...)` currently buffers and sends one S3 PUT with the same cap; multipart upload, SSE-C, and checksum selection are not supported. Conditional requests and range GETs implement the common R2 behavior. `list({ include: [...] })` performs -extra HEAD requests for metadata fields and applies a concurrency cap. +extra HEAD requests for metadata fields and applies a concurrency cap. Tenant-supplied +`Headers` metadata must carry a canonical IMF-fixdate `Expires` value when that header +is present; malformed write metadata is rejected before the host binding call. Tenant-facing R2 errors expose operation/status plus virtual object keys where useful, but not raw S3 response bodies or physical `r2///...` keys. Control-plane R2 admin errors may retain backend detail for operators. @@ -150,7 +156,9 @@ Service bindings are frozen at caller deploy time. Control resolves target names worker, version, and entrypoint, stores them in caller metadata, and runtime loads that exact target version on first use. Promoting the target later does not move existing callers; caller redeploy is the refresh boundary. This version pinning makes rollback -and version delete referrer checks deterministic. +and version delete referrer checks deterministic. Runtime revalidates the persisted +pinned version with the canonical positive JavaScript-safe-integer grammar before +materializing the binding. Cross-namespace service bindings require a target `[[exports]]` entry for the bound entrypoint, including `entrypoint = "default"` for the default export. The entry's @@ -183,9 +191,12 @@ Data-plane bindings use their own storage: Runtime must treat Redis bundle metadata as control-authored, but still revalidates reserved runtime entrypoint and binding names when materializing older stored metadata. -It also fails closed if older metadata contains Python module entries or upstream -experimental compatibility flags, because those would otherwise reach workerd as -opaque cold-load failures under the current stock binary. +It also fails closed if older metadata has a `compatibilityDate` before `2026-04-01`, +contains Python module entries or upstream experimental compatibility flags, disables +WDL's required enhanced error serialization, or explicitly enables that behavior after +it became the workerd default. The date floor and enhanced-serialization requirement +are WDL forward-only policy; the remaining checks prevent unsupported metadata from +reaching the current stock workerd binary as an opaque cold-load failure. ## Ownership / Concurrency / Failure Semantics @@ -196,12 +207,32 @@ opaque cold-load failures under the current stock binary. - Internal active-version scheduled/queue dispatches opt into sibling eviction; frozen workflow dispatches do not. - Wrapper generation hides raw env from unwrapped entrypoints whenever privileged - internal Fetchers are injected. + internal Fetchers are injected. Its host-wrapper runtime evaluates before tenant + modules and captures the intrinsics used to decide handler or env wrapping, so tenant + top-level prototype mutation cannot bypass the env boundary. +- Host facade request ids use an invocation-local `AsyncLocalStorage` store. Runtime + ensures that the loaded worker has ALS support: an existing `nodejs_compat` setup is + left unchanged; otherwise a standalone `no_nodejs_als` is replaced by the narrow + `nodejs_als` flag. Concurrent calls on a persistent Durable Object instance therefore + cannot overwrite one shared context. The wrapper does not inspect, mutate, or replace + tenant return values. Native Promise and `async` continuations retain context; custom + thenables are not a supported request-id propagation boundary. Only the outermost + generated-wrapper invocation establishes the ALS store, including a store whose + request id is null, so ordinary nested wrapper calls inherit it instead of replacing + it with a second id. Tenant code shares the isolate's ambient ALS machinery and can + deliberately bind or restore another ambient frame, so loaded-worker propagation is + best-effort against tenant interference, not a security boundary. Request ids are + untrusted diagnostic metadata and must never authorize, fence, or deduplicate work. + Request-id syntax is owned by the injected canonical request-id module rather than + copied into the wrapper runtime. - Request context wrappers swap facade objects into env and propagate request id where - that event class can carry it. + that event class can carry it. do-runtime alarm and RPC calls enter through private + fetch dispatches carrying the outer request id, so the generated wrapper establishes + invocation-local context without adding platform metadata to tenant method arguments. - An uncaught tenant `fetch()` exception maps to a platform `502 runtime_error` response with the request id. Exception details are emitted to structured logs/live - tail and are not copied into the client body. + tail and are not copied into the client body. Tail formatting cannot replace the + original throwable when its string conversion fails. - Internal scheduled, queue, and workflow dispatch routes use result envelopes for handler outcomes. A tenant handler error is represented as outcome state for the scheduler/workflow protocol, not as a generic platform transport error. diff --git a/docs/modules/runtime.zh.md b/docs/modules/runtime.zh.md index 1249d29..c97ebe5 100644 --- a/docs/modules/runtime.zh.md +++ b/docs/modules/runtime.zh.md @@ -15,6 +15,7 @@ Runtime 从 Redis 加载不可变 worker version,构建 tenant-facing `env` bi - `runtime/dispatch.js` 和 `runtime/dispatch/*`:fetch/scheduled/queue/workflow dispatch helper。 Worker 通过不可变 id `::` 加载。Promote 会生成新的 id,因此 active-version 变化自然触发 fresh isolate cold-load。 +`runtime/load.js` 也统一持有 `workerLoader.get()` wrapper:每次 cache miss 都会登记供后续 eviction 使用,只有 active-version dispatch path 会请求 sibling eviction;service binding 加载 pinned version 时不会驱逐 sibling。 两个 runtime pool 使用同一个 image 和源码,但 capnp config 不同: @@ -40,7 +41,7 @@ Tenant 可见 binding 包括 KV、R2、D1、Durable Objects、Queues、ASSETS、 workerd 提供 isolate、module evaluation、named entrypoint 和 JSRPC 机制。Cloudflare 生产平台通常在外部服务里实现的 binding 后端,则由 WDL 自己补齐。因此 runtime 把 binding 当作 adapter: - KV、queue producer 这类纯数据 binding 调 colocated redis-proxy sidecar。Loaded worker 看到 Cloudflare-shaped object,但 method call 会先通过 workerd JSRPC 回到 runtime,再经 HTTP 调 redis-proxy。 -- Secret value 也在 cold-load 时经过 redis-proxy。redis-proxy 解密 `WDL-ENC:` value 后,runtime 在 internal load envelope 中收到 plaintext `ns_secrets` 和 `worker_secrets`;tenant-facing `env` 形状保持不变。Env materialization 使用固定优先级:bundle vars,然后 namespace secrets,然后 worker secrets。同名 worker-level secret 覆盖 namespace-level secret,namespace-level secret 覆盖 var。Control 会在 deploy 和 secret mutation 过程中用留有 headroom 的预算估算完整 workerLoader env,包括用户 vars/secrets、runtime 注入的 binding/workflow env value,以及 platform/service binding props 中复制的 required caller secret,避免超限配置拖到 runtime cold-load 才失败。 +- Secret value 也在 cold-load 时经过 redis-proxy。redis-proxy 解密 `WDL-ENC:` value 后,runtime 在 internal load envelope 中收到 plaintext `ns_secrets` 和 `worker_secrets`;tenant-facing `env` 形状保持不变。Env materialization 使用固定优先级:bundle vars,然后 namespace secrets,然后 worker secrets。同名 worker-level secret 覆盖 namespace-level secret,namespace-level secret 覆盖 var。Control 会在 deploy 和 secret mutation 过程中用 shape-only factory 调用 cold-load 同一个 `buildWorkerEnv()` materializer,再用留有 headroom 的预算估算完整 workerLoader env,包括用户 vars/secrets、runtime 注入的 binding/workflow env value,以及 platform/service binding props 中复制的 required caller secret,避免超限配置拖到 runtime cold-load 才失败。 - D1、Durable Objects、Workflows 这类 stateful binding 调专门 backend service。Hidden backend Fetcher 留在 runtime 内部,并在 tenant code 观察 `env` 前被删除。 - R2 是 S3-compatible object-storage adapter:runtime 使用平台 credential 签名请求,并发送到配置的 endpoint。 - ASSETS 是 deploy artifact URL helper:control 在 deploy 时把 assets 上传到 S3-compatible storage;runtime 读取 `__meta__.assets` 和 `ASSETS_CDN_BASE`,只暴露 `env.ASSETS.url(path)` 用来生成 tokenized CDN URL。 @@ -52,13 +53,13 @@ KV 是最直接的例子。Runtime 把 `KV` 导出成 named entrypoint,并用 KV 支持常用 `KVNamespace` 调用:`get`、batch `get`、`getWithMetadata`、batch `getWithMetadata`、`put`、`delete` 和 `list`。`get` 支持 text、JSON、arrayBuffer 和 stream 形状;batch read 支持 text 和 JSON。Runtime shim 在 proxy 前把 value 限制到 25 MiB,stream value 也用同一个 cap 读取。所有 KV 操作的 key 都在 redis-proxy 边界限制为 512 个 UTF-8 字节,包括 list prefix 和 batch read。`list()` 基于 Redis `HSCAN`,不是 Cloudflare 有序 B-tree:key 不排序,cursor 是 opaque WDL cursor,并发写可能乱序出现或被再次看到。`limit` 上限是 1000。`cacheTtl` 只是 API shape;没有 Cloudflare edge read cache 或 global eventual-consistency window。 -R2 binding 把 `bucket_name` 映射到平台 S3-compatible bucket 下的 namespace-scoped virtual bucket:`r2///`。同一个 namespace 中使用同一 `bucket_name` 的 worker 会有意共享数据;不同 namespace 通过前缀隔离。Runtime 支持常用 `head`、`get`、`put`、`delete` 和 `list` 路径。`get()` 返回 streaming body,便捷 reader 执行 25 MiB cap。`put(stream, ...)` 目前会先 buffer,再发单个 S3 PUT,并使用同一个 cap;不支持 multipart upload、SSE-C 和 checksum selection。Conditional requests 和 range GET 实现常用 R2 行为。`list({ include: [...] })` 为 metadata fields 额外执行 HEAD,并使用并发 cap。Tenant-facing R2 error 只暴露 operation/status,以及有帮助的 virtual object key;不会暴露原始 S3 response body 或 physical `r2///...` key。Control-plane R2 admin error 可以为 operator 保留 backend detail。 +R2 binding 把 `bucket_name` 映射到平台 S3-compatible bucket 下的 namespace-scoped virtual bucket:`r2///`。同一个 namespace 中使用同一 `bucket_name` 的 worker 会有意共享数据;不同 namespace 通过前缀隔离。Runtime 支持常用 `head`、`get`、`put`、`delete` 和 `list` 路径。`get()` 返回 streaming body,便捷 reader 执行 25 MiB cap。`put(stream, ...)` 目前会先 buffer,再发单个 S3 PUT,并使用同一个 cap;不支持 multipart upload、SSE-C 和 checksum selection。Conditional requests 和 range GET 实现常用 R2 行为。`list({ include: [...] })` 为 metadata fields 额外执行 HEAD,并使用并发 cap。Tenant 提供的 `Headers` metadata 如果包含 `Expires`,其值必须是 canonical IMF-fixdate;malformed write metadata 会在 host binding call 前被拒绝。Tenant-facing R2 error 只暴露 operation/status,以及有帮助的 virtual object key;不会暴露原始 S3 response body 或 physical `r2///...` key。Control-plane R2 admin error 可以为 operator 保留 backend detail。 ASSETS 是 deploy-artifact helper,不是完整 Cloudflare Pages asset pipeline。Control 把文件上传到 `assets////`,注入 `ASSETS` binding,runtime 暴露同步的 `env.ASSETS.url(path)`。该方法在 runtime 中不做 IO,并用 `ASSETS_CDN_BASE` 返回浏览器可访问的 CDN URL。Path 按 `/` 切段,空段、`.` 和 `..` 被拒绝,每段会 percent-encode。Version 在 load 时绑定,因此 rollback 会切换 asset URL。需要对静态文件做 auth 或 rewrite 的 worker 应把文件留在 bundle 里,而不是使用 declared `assets`。 R2 和 ASSETS 生命周期语义故意不同。ASSETS 是 deploy artifact,version/worker delete 会 stage `worker-delete-s3-cleanup` work。R2 是 tenant runtime data,worker delete 永远不删除 R2 object。 -Service binding 在 caller deploy 时冻结。Control 解析 target namespace、worker、version 和 entrypoint,存入 caller metadata,runtime 第一次使用时加载该精确 target version。之后 target promote 不会移动已有 caller;caller redeploy 才是 refresh 边界。这种 version pinning 让 rollback 和 version delete referrer check 都是确定的。 +Service binding 在 caller deploy 时冻结。Control 解析 target namespace、worker、version 和 entrypoint,存入 caller metadata,runtime 第一次使用时加载该精确 target version。之后 target promote 不会移动已有 caller;caller redeploy 才是 refresh 边界。这种 version pinning 让 rollback 和 version delete referrer check 都是确定的。Runtime 会在 materialize binding 前用 canonical positive JavaScript-safe-integer grammar 重新校验 persisted pinned version。 Cross-namespace service binding 要求 target 为被绑定的 entrypoint 声明 `[[exports]]` entry;default export 也要声明 `entrypoint = "default"`。该 entry 的 `allowed_callers` 控制 cross-namespace 访问;`["*"]` 对所有 namespace 开放,空数组则关闭 cross-namespace caller。没有 `[[exports]]` 的 target 只向 same-namespace caller 暴露 default entrypoint。Same-namespace caller 绕过 ACL,但 target 一旦声明 `[[exports]]`,仍受 strict entrypoint visibility 约束。ACL 变化是 deploy-time,不是 call-time;既有 caller 会保持 pin,直到 redeploy。 @@ -74,16 +75,17 @@ Runtime 通过 `redis-proxy` 从 DB 0 读取不可变 bundle 和 metadata。Data - D1 和 DO binding 调用专门 runtime service。 - R2/ASSETS 使用 S3-compatible object storage。 -Runtime 可以把 Redis bundle metadata 视为 control-authored,但 materialize 旧 metadata 时仍会重新校验 reserved runtime entrypoint 和 binding name。旧 metadata 如果包含 Python module entry 或上游 experimental compatibility flag,也会 fail closed;否则这些形态会在当前 stock workerd binary 下变成 opaque cold-load failure。 +Runtime 可以把 Redis bundle metadata 视为 control-authored,但 materialize 旧 metadata 时仍会重新校验 reserved runtime entrypoint 和 binding name。旧 metadata 如果包含早于 `2026-04-01` 的 `compatibilityDate`、Python module entry、上游 experimental compatibility flag、禁用 WDL 要求的 enhanced error serialization,或在该行为已成为 workerd 默认值后仍显式启用对应 flag,也会 fail closed。日期下限与 enhanced serialization 要求是 WDL 的 forward-only policy;其余检查避免 unsupported metadata 直到当前 stock workerd cold-load 才以 opaque error 失败。 ## Ownership / 并发 / 失败语义 - workerLoader cache 没有 LRU。Runtime 给每个 loaded worker 注入 `__WdlAbort__`,并在 active-version cold-load 时 evict sibling historical versions。 - Service-binding cold load 会记录 loaded version,但不 evict sibling,因为 service binding 可能有意指向 frozen historical version。 - Internal active-version scheduled/queue dispatch 会 opt into sibling eviction;frozen workflow dispatch 不会。 -- 只要注入 privileged internal Fetcher,wrapper generation 就会避免 raw env 暴露给未包装 entrypoint。 -- Request context wrapper 会把 facade object 换进 env,并在事件类型允许时传播 request id。 -- Tenant `fetch()` 未捕获异常会映射为平台 `502 runtime_error` response,并带 request id。异常细节输出到结构化日志/live tail,不复制进客户端 body。 +- 只要注入 privileged internal Fetcher,wrapper generation 就会避免 raw env 暴露给未包装 entrypoint。Host-wrapper runtime 会在 tenant module 前执行,并捕获参与 handler/env wrapping 决策的 intrinsic,因此 tenant 顶层 prototype mutation 不能绕过 env 边界。 +- Host facade request id 使用 invocation-local `AsyncLocalStorage` store。已有 `nodejs_compat` 配置会保持不变;否则 Runtime 会把 standalone `no_nodejs_als` 替换为范围更窄的 `nodejs_als` flag。持久 Durable Object 实例上的并发调用不会再覆盖同一个共享 context,wrapper 也不会检查、修改或替换 tenant 返回值。Native Promise 和 `async` continuation 会保留 context;custom thenable 不属于受支持的 request-id propagation boundary。只有最外层 generated-wrapper invocation 会建立 ALS store,即使该 store 的 request id 为 null,因此普通嵌套 wrapper 调用会继承它而不是写入第二个 id。Tenant code 与 host 共用 isolate 的 ambient ALS 机制,可以主动 bind 或恢复另一个 ambient frame,因此 loaded-worker propagation 面对 tenant 干预时只是 best-effort,不是安全边界。Request id 是不可信的诊断 metadata,不能用于授权、fence 或去重。Request-id 语法由 injected canonical request-id module 拥有,不在 wrapper runtime 内复制。 +- Request context wrapper 会把 facade object 换进 env,并在事件类型允许时传播 request id。do-runtime alarm 和 RPC 通过携带外层 request id 的私有 fetch dispatch 进入,使 generated wrapper 建立 invocation-local context,而不会把平台 metadata 加入 tenant method argument。 +- Tenant `fetch()` 未捕获异常会映射为平台 `502 runtime_error` response,并带 request id。异常细节输出到结构化日志/live tail,不复制进客户端 body;throwable 自身无法转成字符串时,tail formatting 也不能反向覆盖原始异常。 - Internal scheduled、queue 和 workflow dispatch route 使用 result envelope 表达 handler outcome。Tenant handler error 是 scheduler/workflow 协议中的 outcome state,不是 generic platform transport error。 - Runtime 没有 route-cache invalidation protocol。`workerLoader` cache key 是不可变 worker id,因此 promote 后的新 version 是新 key,会自然 cold-load。 - `runtime/tail-worker.js` 通过 `workerCode.tails` 附加到每次 dynamic load。它总是输出结构化 stdout;只有 shared tail forwarder 看到 active subscription 后,才转发到 `wdl tail`。 diff --git a/docs/modules/workflows.md b/docs/modules/workflows.md index a4ea74f..2d07fc1 100644 --- a/docs/modules/workflows.md +++ b/docs/modules/workflows.md @@ -82,7 +82,9 @@ Internal: Workflows exclusively owns Valkey DB 2 for instance execution state. Control owns `wf:defs::` in DB 0 for deploy-time workflow key allocation and stable -identity. +identity. The hash retains retired names until whole-worker delete. Definition listing +enumerates that retired history for currently active workers; deploy and single-workflow +status/lifecycle paths read only the names they need. Key concepts: @@ -117,6 +119,7 @@ Key families: | `wf:by-worker::` | Set | workflows | Instance discovery by worker. | Used by list/delete checks; entries are removed by retention/delete cleanup. | | `wf:by-workflow:::` | ZSET | workflows | Per-workflow instance list index ordered for bounded pagination. | Retention/delete cleanup removes the sorted-set member. | | `wf:by-version:::` | Set | workflows | Frozen-version referrer index. | Blocks version delete while live instances reference the version. | +| `wf:pending-version:::` | ZSET | workflows | Short-lived restart target-version blockers, scored by expiry time. | Version-delete checks active members; restart renews only a live member and atomically validates it before creating the durable `wf:by-version` referrer. Members expire after 30 seconds, and the ZSET has a refreshed 60-second key TTL for physical cleanup. | | `wf:retention` | ZSET | workflows | Terminal retention due index. | Retention tick deletes expired terminal instances. | | `wf:internal:do-alarm:{}:state` | Hash | workflows | Authoritative backend job state for one Durable Object SQLite alarm row. | Successful delivery, retry exhaustion, explicit delete, and worker cleanup remove the job. | | `wf:internal:do-alarm:due:` | ZSET | workflows | DO alarm due index. Score is due timestamp in milliseconds. | Tick promotion moves eligible jobs to ready. | @@ -128,11 +131,25 @@ Key families: - Workflows are same-worker only in V2. - Instances freeze the worker version/class identity they were created with. +- Control fails closed on malformed active workflow entries and malformed `wf:defs` + records encountered by an operation; management paths return `corrupt_meta`, while + deploy returns `workflow_definition_corrupt` when reusing a damaged historical + definition. Damaged authoritative metadata is not exposed as a normal missing or + retired workflow. Normal deploy and single-workflow paths do not scan unrelated + historical definitions. - Scheduler only wakes workflows; workflows owns admission, fairness, shard ticks, ready/due movement, and runtime dispatch. - Scheduler also wakes Workflows-owned internal DO alarm jobs through the same `/internal/workflows/tick` endpoint; scheduler never reads or writes DO alarm state directly. +- Workflows rejects non-canonical DO alarm identity before persisting jobs, revalidates + persisted alarm identity before dispatch, and validates an active route + version before using it as a retarget. Namespace, worker, and version checks reuse + `wdl-rust-common`; do-runtime protocol grammar and identity helpers own the canonical + alarm-specific fields and aggregate 512-byte DO host-id contract. Workflows mirrors + and revalidates that contract before persistence and dispatch. + Runtime run dispatch and progress callbacks share one system-vs-user runtime endpoint + selector inside the workflows crate. - 32 scheduling shards partition ready/due work. - Ready tokens are deduplicated hints; instance hash state is authority. - Execution commits are fenced by `generation`, `runToken`, active instance status, and @@ -154,9 +171,13 @@ Key families: (`step.sleep`, `step.sleepUntil`, `step.waitForEvent`) remain exclusive and must not overlap another in-flight step because they suspend the whole workflow run. - Termination is an explicit non-success terminal outcome and uses error retention. -- Per-instance aggregate payload cap is 16 MiB. Step/event over-cap writes fail the - request; over-cap runtime terminal results transition the instance to failed in the - same transaction. +- `Workflow.createBatch()` accepts at most 100 entries per call. Runtime prevalidation + and Rust admission share this pinned limit. +- A single workflow result is capped at 1 MiB and a runtime-to-workflows backend JSON + request at 2 MiB. Runtime prevalidation and the Rust backend share the pinned + `workflow_payload_too_large` contract. The per-instance aggregate payload cap is + 16 MiB. Step/event over-cap writes fail the request; over-cap runtime terminal + results transition the instance to failed in the same transaction. - Workflows semantic request caps use `request_too_large`; this is distinct from HTTP-body parser `request_body_too_large` in control/runtime protocols. Workflow errors otherwise use the platform `{ error, message }` envelope on HTTP boundaries. @@ -178,6 +199,13 @@ canonicalizes against the current active route before writing DB 2, so new durab business processes start on the active version. Existing instances replay against their stored `frozenVersion`; promotion does not change their code. Worker-version delete is blocked by `wf:by-version` while non-expired instances still reference the version. +Before restart revalidates the active export, it publishes a short-lived target-version +blocker. Its final DB 2 transition atomically creates the durable referrer and removes +that blocker, so version delete cannot pass between active-version resolution and the +restart commit. +Runtime validates every dispatched `frozenVersion` with the same positive +JavaScript-safe-integer version parser used by bundle keys; malformed persisted tags +fail before worker loading. Scheduling is hint-based but state-authoritative: diff --git a/docs/modules/workflows.zh.md b/docs/modules/workflows.zh.md index 1e57d35..70418c7 100644 --- a/docs/modules/workflows.zh.md +++ b/docs/modules/workflows.zh.md @@ -44,7 +44,7 @@ Control / CLI: ## Redis / Storage 合同 -Workflows 独占 Valkey DB 2 作为 instance execution state。Control 在 DB 0 拥有 `wf:defs::`,用于 deploy-time workflow key allocation 和稳定 identity。 +Workflows 独占 Valkey DB 2 作为 instance execution state。Control 在 DB 0 拥有 `wf:defs::`,用于 deploy-time workflow key allocation 和稳定 identity。该 Hash 会保留 retired name 直到 whole-worker delete;definition list 只会为当前 active worker 枚举这段 retired history,而 deploy 和单个 workflow 的 status/lifecycle 路径只读取自己需要的 name。 关键概念: @@ -70,6 +70,7 @@ Key families: | `wf:by-worker::` | Set | workflows | 按 worker 发现 instance 的索引。 | list/delete check 使用;retention/delete cleanup 移除 entry。 | | `wf:by-workflow:::` | ZSET | workflows | 按 workflow key 分页列 instance 的有序索引。 | retention/delete cleanup 删除 sorted-set member。 | | `wf:by-version:::` | Set | workflows | frozen-version referrer index。 | live instance 仍引用该 version 时阻止 version delete。 | +| `wf:pending-version:::` | ZSET | workflows | 按过期时间计分的短期 restart target-version blocker。 | Version-delete 检查 active member;restart 只续租未过期 member,并在创建持久 `wf:by-version` referrer 前原子复核。Member 30 秒后过期,ZSET key 使用随写入刷新的 60 秒 TTL 做物理回收。 | | `wf:retention` | ZSET | workflows | terminal retention due index。 | Retention tick 删除过期 terminal instance。 | | `wf:internal:do-alarm:{}:state` | Hash | workflows | 单个 Durable Object SQLite alarm row 的 backend job 权威状态。 | 成功 delivery、retry 耗尽、显式 delete 和 worker cleanup 会移除 job。 | | `wf:internal:do-alarm:due:` | ZSET | workflows | DO alarm due index。score 是 due timestamp milliseconds。 | Tick promotion 把到期 job 移到 ready。 | @@ -81,15 +82,18 @@ Key families: - V2 workflow 只支持 same-worker。 - Instance 冻结创建时的 worker version/class identity。 +- Control 会对当前操作实际读到的 malformed active workflow entry 和 malformed `wf:defs` record fail closed;管理路径返回 `corrupt_meta`,deploy 在复用损坏的历史 definition 时返回 `workflow_definition_corrupt`。损坏的权威 metadata 不会被暴露为正常的 missing 或 retired workflow。正常 deploy 和单个 workflow 路径不会扫描无关的历史 definition。 - Scheduler 只负责唤醒 workflows;admission、fairness、shard tick、ready/due movement 和 runtime dispatch 都由 workflows 负责。 - Scheduler 也通过同一个 `/internal/workflows/tick` endpoint 唤醒 Workflows-owned internal DO alarm jobs;scheduler 不直接读写 DO alarm state。 +- Workflows 在持久化 DO alarm job 前拒绝 non-canonical alarm identity,在 dispatch 前重新校验持久化 alarm identity,并在把 active route 用作 retarget 前校验其 version。其中 namespace、worker、version 校验复用 `wdl-rust-common`;do-runtime protocol grammar 与 identity helper 拥有 canonical alarm-specific field 和 aggregate 512-byte DO host-id 合同,Workflows 在持久化和 dispatch 前镜像并重新校验该合同。Runtime run dispatch 与 progress callback 在 workflows crate 内共用同一个 system-vs-user runtime endpoint selector。 - 32 个 scheduling shards 划分 ready/due work。 - Ready token 是去重 hint;instance hash state 是权威状态。 - Execution commit 同时用 `generation`、`runToken`、active instance status 和未过期 run lease fence。Step commit/register 接受同一 run 的 `running` 或 `waiting` 状态,因此一个并行 sibling 进入 retry/wait 后,另一个 sibling 仍可完成;completed runtime terminal 要求 `running`,failed runtime terminal 在 run lease 仍有效时也可以关闭由非法未 await suspending step 造成的同一 run `waiting` 状态。如果 lease 已过期,workflows 只恢复 ready hint,让下一次 claim 在新 lease 下 replay。Lifecycle commit 只用 generation fence,并在同一个 Lua commit 内 rotate `generation`。 - Runtime replay cache 只是 advisory。DB 2 step state 是权威。 - Runtime 可以并发发起多个 `step.do`,常见形式是 `Promise.all`;每次调用按用户代码调用顺序分配 deterministic ordinal,从当前已完成 step frontier 记录 DAG dependencies,并在 run fence 下独立 commit。`step.do` callback 不能启动另一个 workflow step,即使在 callback 的 `await` 之后也不允许;并行 sibling promise 应在 run body 中、callback 代码进入 in-flight 之前创建。如果 run 在已启动 step settle 前返回,会按 invalid run 失败,所以用户代码必须 await 并发 step promise。Suspending operation(`step.sleep`、`step.sleepUntil`、`step.waitForEvent`)仍保持互斥,不能和其它 in-flight step 重叠,因为它们会 suspend 整个 workflow run。 - Termination 是显式 non-success terminal outcome,使用 error retention。 -- 每个 instance 的 aggregate payload cap 是 16 MiB。Step/event 超 cap 写入会让请求失败;runtime terminal result 超 cap 会在同一事务内把 instance 转成 failed。 +- `Workflow.createBatch()` 每次调用最多接受 100 项;Runtime prevalidation 与 Rust admission 共享这项 pinned limit。 +- 单个 workflow result 的上限是 1 MiB,runtime-to-workflows backend JSON request 的上限是 2 MiB。Runtime prevalidation 和 Rust backend 共享 pinned `workflow_payload_too_large` contract。每个 instance 的 aggregate payload cap 是 16 MiB。Step/event 超 cap 写入会让请求失败;runtime terminal result 超 cap 会在同一事务内把 instance 转成 failed。 - Workflows 语义 request cap 使用 `request_too_large`;它不同于 control/runtime 协议中的 HTTP-body parser `request_body_too_large`。除此之外,HTTP 边界上的 workflow error 使用平台 `{ error, message }` envelope。Client-facing proxy 应把 workflows 5xx 当作 backend/platform failure,不应依赖 response body 中的 raw backend diagnostic message。 Workflow execution 使用两条 channel: @@ -97,7 +101,7 @@ Workflow execution 使用两条 channel: 1. Loaded worker 通过 reserved `__WDL_WORKFLOWS_BACKEND__` Fetcher binding 调 workflows。Runtime 从 bundle metadata 附加 identity;workflows 不信任 tenant body 中的 namespace、worker、version、workflow key、class 或 instance identity。 2. workflows 把已 claim 的 run dispatch 回 runtime `:8088` 上的 `/internal/workflows/run`。Runtime 加载 frozen worker version 并调用 `className.run(event, stepFacade)`。 -Create/restart 与 replay 的 version pinning 不同。新的 `create()` 或 `restart()` 写 DB 2 前会按当前 active route canonicalize,因此新的 durable business process 从 active version 开始。已有 instance 使用自己存的 `frozenVersion` replay;promotion 不会改变它的代码。只要 non-expired instance 仍引用某个 version,`wf:by-version` 就会阻止 worker-version delete。 +Create/restart 与 replay 的 version pinning 不同。新的 `create()` 或 `restart()` 写 DB 2 前会按当前 active route canonicalize,因此新的 durable business process 从 active version 开始。已有 instance 使用自己存的 `frozenVersion` replay;promotion 不会改变它的代码。只要 non-expired instance 仍引用某个 version,`wf:by-version` 就会阻止 worker-version delete。Restart 在重新校验 active export 前写入一个短期 target-version blocker,最终 DB 2 transition 会原子地建立持久 `wf:by-version` referrer 并删除该 blocker,因此 version delete 不能从 active-version resolution 和 restart commit 之间穿过。Runtime 会用 bundle key 共用的正 JavaScript-safe-integer version parser 校验每个 dispatch 的 `frozenVersion`;malformed persisted tag 会在加载 worker 前失败。 Scheduling 是 hint-based,但状态权威在 instance hash: diff --git a/docs/redis-key-layout.md b/docs/redis-key-layout.md index 7c4119a..8313e4d 100644 --- a/docs/redis-key-layout.md +++ b/docs/redis-key-layout.md @@ -8,14 +8,14 @@ families, and ownership rules that span modules. WDL uses a deliberate logical split: -- **DB 0, control plane:** bundles, routes/patterns, auth, D1/DO owner state, cron +- **`DB 0`, control plane:** bundles, routes/patterns, auth, D1/DO owner state, cron config, queue-consumer config, lifecycle metadata, and workflow definitions (`wf:defs:*`). -- **DB 1, data plane:** KV hash buckets, queue streams, delayed queues, orphan streams, +- **`DB 1`, data plane:** KV hash buckets, queue streams, delayed queues, orphan streams, and live log-tail streams. -- **DB 2, workflows:** `wf:schema_version`, instance state, step records/summaries, - ready/due shards, events and event-type indexes, payload refs, retention indexes, and - run leases. +- **`DB 2`, workflows:** `wf:schema_version`, instance state, step records/summaries, + ready/due shards, events and event-type indexes, payload refs, retention indexes, + restart target-version blockers, and run leases. Local compose, Kubernetes, and Terraform enable this split. Rust services and the Rust `redis-proxy` use `DATA_REDIS_URL` / `DATA_REDIS_DB` to select the @@ -36,7 +36,9 @@ workers: Set, worker names with worker-owned lifecycle st worker:::next_version String, monotonic version counter, survives delete worker-versions:: ZSET, score=int version, member="v" worker:::v: Hash, bundle bytes plus __meta__ -worker-delete-lock:: String EX 30, per-worker delete critical-section lock +worker-delete-lock:: String EX 30, per-worker delete critical-section lock; + value is whole: or version:; DO first-owner + claim WATCHes it and only whole fences ownership worker-version-referrers::: Set, canonical JSON version-pinned caller refs hosts: Set, declared operator host intent @@ -52,8 +54,9 @@ secrets: Hash, namespace-level WDL-ENC envelopes secrets:: Hash, worker-level WDL-ENC envelopes ``` -`worker:::v:` uses the integer version in the key, not the `"v"` -tag. Test fixtures that seed Redis directly must use `shared/version.js#bundleKey`. +`worker:::v:` uses a positive JavaScript-safe integer version in the +key, not the `"v"` tag. Test fixtures that seed Redis directly must use +`shared/version.js#bundleKey`. `namespaces` is an active worker gate. It is populated when a namespace has an active worker route and may be removed when the last active worker is deleted. @@ -71,8 +74,8 @@ scheduled version is no longer retained. A key-grammar change must update the JS helper, the Rust helper, and every reader together. `workers:` means the worker has worker-owned lifecycle state: retained bundle, -active projection, or worker-level secrets. Secret-only workers are intentionally listed -and whole-deletable. +active projection, worker-level secrets, or workflow definitions. Secret-only and +definitions-only workers are intentionally listed and whole-deletable. ## Route And Host Projection @@ -81,7 +84,8 @@ then reads `patterns:` and uses the slot value's embedded `version` to con `x-worker-id` without consulting `routes:`. Pattern slot values are compact `v2\t\t\t\t\t` records encoded by `shared/route-projection.js`, not JSON. Promote updates both projections in the same -Redis transaction. +Redis transaction. Control mutation and delete paths fail closed on a nonempty slot that +cannot be decoded; they do not treat an unknown owner as an empty slot. `hosts:` is operator intent: the namespace is allowed to use those hosts. `declared-hosts` is a gateway gate for hosts declared by at least one namespace. @@ -118,6 +122,15 @@ not base64. Typical fields include: } ``` +Control writes `__meta__` as a JSON object. Control-plane consumers parse required +bundle metadata through `control/lib.js::parseBundleMeta()`; malformed JSON, arrays, +and scalar values fail closed as `corrupt_meta`. Absence remains use-site-specific. +Paths that need metadata to compute a correct projection change, uniqueness proof, +lifecycle cleanup, workflow view, or environment budget fail closed while their +authoritative route or index still names the bundle. Deploy discovery/link preflight +does not classify absence as `corrupt_meta`; the watched commit remains authoritative +and rejects a missing pinned service target as `target_drift`. + Routes, crons, queue consumers, bindings, vars, exports, workflow definitions, and asset prefixes are version metadata. Rollback is a promote of an older immutable version. @@ -135,6 +148,11 @@ Feature modules own the detailed contracts: Cross-cutting constraints: +- Persisted D1/DO owner records must reconstruct the encoded scope of the Redis key + under which they are read. A syntactically valid record stored under another scope + fails closed before forwarding, takeover, renewal, or release. DO owner resolution + also binds the record's canonical namespace and worker to the invoking bundle before + reading that bundle's active storage pointer. - Indexes are usually repairable projections, not authority. The module doc must state which key is authoritative before adding a writer. - Lifecycle and delete blocker indexes are authoritative where the module says they are; @@ -147,6 +165,11 @@ Cross-cutting constraints: - Workflows owns DB 2 instance state. `wf:ready:cursor` is the internal ready-shard fairness cursor. Control owns only DB 0 `wf:defs:*`; other tiers must not write DB 2 directly. +- `wf:pending-version:::` is a Workflows-owned, 30-second restart + blocker. Version-delete checks it with `wf:by-version`; renewals cannot resurrect an + expired member, and the successful-restart DB 2 script revalidates the lease before + replacing it with the durable version referrer. The ZSET key has a refreshed + 60-second TTL so abandoned marker keys are physically reclaimed. - Workflows also owns internal DB 2 `wf:internal:do-alarm:*` jobs for Durable Object alarm backend scheduling. do-runtime writes alarms through the workflows HTTP API instead of writing those keys directly. `wf:internal:do-alarm:ready:cursor` is the diff --git a/docs/redis-key-layout.zh.md b/docs/redis-key-layout.zh.md index 71e8921..2efe666 100644 --- a/docs/redis-key-layout.zh.md +++ b/docs/redis-key-layout.zh.md @@ -6,9 +6,9 @@ WDL 使用明确的逻辑切分: -- **DB 0,控制面:**bundle、routes/patterns、auth、D1/DO owner state、cron config、queue-consumer config、lifecycle metadata,以及 workflow definition(`wf:defs:*`)。 -- **DB 1,数据面:**KV hash bucket、queue stream、delayed queue、orphan stream 和 live log-tail stream。 -- **DB 2,workflows:**`wf:schema_version`、instance state、step record/summary、ready/due shard、event 和 event-type index、payload ref、retention index、run lease。 +- **`DB 0`,控制面:**bundle、routes/patterns、auth、D1/DO owner state、cron config、queue-consumer config、lifecycle metadata,以及 workflow definition(`wf:defs:*`)。 +- **`DB 1`,数据面:**KV hash bucket、queue stream、delayed queue、orphan stream 和 live log-tail stream。 +- **`DB 2`,workflows:**`wf:schema_version`、instance state、step record/summary、ready/due shard、event 和 event-type index、payload ref、retention index、restart target-version blocker、run lease。 Local compose、Kubernetes 和 Terraform 都启用这个切分。Rust service 和 Rust `redis-proxy` 使用 `DATA_REDIS_URL` / `DATA_REDIS_DB` 选择 data-plane Redis connection/database;嵌入的 JS control/log-tail 路径使用 `DATA_REDIS_ADDR` 加 `DATA_REDIS_DB`,因为它们的 RESP client 接收 host:port address。未设置这些 data-plane 变量的部署会把数据面 key 留在 control Redis connection/database,直到显式 opt in。Workflows 不同:未设置 `WORKFLOWS_REDIS_URL` 时 workflows service 仍默认使用 DB 2;只有显式设置 `WORKFLOWS_REDIS_DB=0` 时才使用 DB 0。 @@ -21,7 +21,7 @@ workers: Set, 有 worker-owned lifecycle state 的 worker worker:::next_version String, 单调 version counter,delete 后保留 worker-versions:: ZSET, score=int version, member="v" worker:::v: Hash, bundle bytes + __meta__ -worker-delete-lock:: String EX 30, 每个 worker 的 delete critical-section lock +worker-delete-lock:: String EX 30, 每个 worker 的 delete critical-section lock;value 是 whole: 或 version:;DO first-owner claim 会 WATCH,且只有 whole 阻止 ownership worker-version-referrers::: Set, canonical JSON 的 version-pinned caller ref hosts: Set, operator 声明的 host intent @@ -37,17 +37,17 @@ secrets: Hash, namespace-level WDL-ENC envelope secrets:: Hash, worker-level WDL-ENC envelope ``` -`worker:::v:` 的 key 使用整数 version,而不是 `"v"` tag。直接 seed Redis 的测试 fixture 必须使用 `shared/version.js#bundleKey`。 +`worker:::v:` 的 key 使用 JavaScript safe-integer 范围内的正整数 version,而不是 `"v"` tag。直接 seed Redis 的测试 fixture 必须使用 `shared/version.js#bundleKey`。 `namespaces` 是 active worker gate。有 active worker route 时会加入,最后一个 active worker 删除时可能移除。Namespace-level secrets 和 data-plane state 等资源可以比这个 set membership 活得更久。Auth 在 delegated token issue 时只把它作为 generated-namespace collision 的 best-effort 信号读取,而不是永久 namespace registry。 `routes:` 和 `worker-versions::` 只能通过 `shared/version.js#routesKey` / `#workerVersionsKey`(以及它们的 Rust 镜像 `rust/common/src/version.rs#routes_key` / `#worker_versions_key`)构造。Control 是唯一 writer;sanctioned reader 是 gateway(route resolution)和 workflows。workflows 有两条读取路径:workflow create / verify 时的 active-export resolution,以及 fired alarm 的 scheduled version 已不再 retained 时的 internal DO alarm retarget。改 key 语法时必须同时更新 JS helper、Rust helper 和所有 reader。 -`workers:` 表示这个 worker 有 worker-owned lifecycle state:retained bundle、active projection 或 worker-level secrets。Secret-only worker 会被有意列出,并可以 whole-delete。 +`workers:` 表示这个 worker 有 worker-owned lifecycle state:retained bundle、active projection、worker-level secrets 或 workflow definitions。Secret-only 和 definitions-only worker 会被有意列出,并可以 whole-delete。 ## Route 和 Host Projection -Subdomain routing 读取 `routes:`。Pattern routing 先检查 `declared-hosts`,再读取 `patterns:`,并使用 slot value 中嵌入的 `version` 构造 `x-worker-id`,不再查 `routes:`。Pattern slot value 是由 `shared/route-projection.js` 编码的紧凑 `v2\t\t\t\t\t` record,不再是 JSON。Promote 在同一个 Redis transaction 中更新两套 projection。 +Subdomain routing 读取 `routes:`。Pattern routing 先检查 `declared-hosts`,再读取 `patterns:`,并使用 slot value 中嵌入的 `version` 构造 `x-worker-id`,不再查 `routes:`。Pattern slot value 是由 `shared/route-projection.js` 编码的紧凑 `v2\t\t\t\t\t` record,不再是 JSON。Promote 在同一个 Redis transaction 中更新两套 projection。Control mutation 和 delete 路径遇到无法 decode 的非空 slot 时会 fail closed,不会把未知 owner 当成空槽。 `hosts:` 是 operator intent:这个 namespace 被允许使用这些 host。`declared-hosts` 是 gateway 对“至少被一个 namespace 声明过的 host”的 gate。`host-declarations:` 记录声明该 host 的 namespace,因此一个 namespace 移除声明时,不会在另一个 namespace 仍声明该 host 的情况下清掉全局 gate。`POST /reload` 会先从 `hosts:` 重建这两个声明索引,再发布 gateway cache invalidation;这给 operator-managed host declaration 提供显式 repair/backfill 路径。`ns-hosts:` 是 active reverse index:这个 namespace 当前在这些 host 上拥有至少一个 slot。`hosts:` 应是 superset。Host reconcile 会先用 `ns-hosts:` 做 fast path,再扫描 `patterns:`。 @@ -73,6 +73,8 @@ Pattern `slot` 是原始 wrangler pattern,例如 `/mcp` 或 `/mcp/*`;它也 } ``` +Control 把 `__meta__` 写为 JSON object。Control-plane consumer 通过 `control/lib.js::parseBundleMeta()` 解析必需的 bundle metadata;malformed JSON、array 和 scalar 值都会以 `corrupt_meta` fail closed。Bundle 缺失的语义仍由具体 use site 持有:当 projection 变更、唯一性证明、lifecycle cleanup、workflow view 或 environment budget 必须消费 metadata 才能产生正确结果时,只要权威 route 或 index 仍指向该 bundle,缺失就 fail closed。Deploy discovery/link preflight 不把缺失归类为 `corrupt_meta`;watched commit 仍是权威检查,并以 `target_drift` 拒绝缺失的 pinned service target。 + Routes、crons、queue consumers、bindings、vars、exports、workflow definitions 和 asset prefixes 都是 version metadata。Rollback 本质上是 promote 一个旧的 immutable version。 ## Feature Key Families @@ -89,9 +91,11 @@ Routes、crons、queue consumers、bindings、vars、exports、workflow definiti 跨模块约束: +- Persisted D1/DO owner record 必须能重建出读取它的 Redis key 所编码的 scope。语法合法但错放在其他 scope 下的记录会在 forwarding、takeover、renew 或 release 前 fail closed;DO owner resolution 还必须在读取 invoking bundle 的 active storage pointer 前,把 record 的 canonical namespace 和 worker 绑定到该 bundle。 - Index 通常是可修复 projection,不是 authority。新增 writer 前,模块文档必须说明哪个 key 是权威状态。 - Lifecycle 和 delete blocker index 在模块文档声明为权威时就是权威;不要增加绕过这些 index 的 request-path fallback scan。 - Queue main stream 不做 trim,因为 at-least-once delivery 是合同。DLQ、orphan、log-tail 这类诊断 stream 可以使用有界 approximate trim。 - Secret hash value 在 steady state 下是 `WDL-ENC:` envelope。`/runtime/load` 没有 plaintext fallback。 - Workflows 拥有 DB 2 instance state。`wf:ready:cursor` 是内部 ready-shard 公平性 cursor。Control 只拥有 DB 0 的 `wf:defs:*`;其他 tier 不应直接写 DB 2。 +- `wf:pending-version:::` 是 Workflows-owned、30 秒的 restart blocker。Version-delete 会将它与 `wf:by-version` 一起检查;续租不能复活已过期 member,restart 成功的 DB 2 script 会在创建持久 version referrer 前再次验证 lease。ZSET key 使用随写入刷新的 60 秒 TTL,确保遗留 marker key 会被物理回收。 - Workflows 还拥有 DB 2 中的 internal `wf:internal:do-alarm:*` jobs,用于 Durable Object alarm backend scheduling。do-runtime 通过 workflows HTTP API 写 alarm,而不是直接写这些 key。`wf:internal:do-alarm:ready:cursor` 是内部 ready-shard 公平性 cursor,不是租户状态。 diff --git a/docs/rust-sidecar-standards.md b/docs/rust-sidecar-standards.md index b657e8d..6860745 100644 --- a/docs/rust-sidecar-standards.md +++ b/docs/rust-sidecar-standards.md @@ -156,6 +156,10 @@ wrapper, and UTF-8-safe string truncation. It must not become a dumping ground f service behavior. Axum-facing helpers are feature-gated so non-HTTP users such as the D1/DO supervisor binaries do not pay for the HTTP stack. +The `test-support` feature exposes the single process-environment override helper used +by Rust service tests. It serializes overrides across modules in one test process and +restores all values during unwinding; production dependency builds leave it disabled. + Local explicit code is still preferable when sidecars have different behavior around: - service-specific shutdown/drain timing and shutdown log events diff --git a/docs/rust-sidecar-standards.zh.md b/docs/rust-sidecar-standards.zh.md index 762cc8b..811d459 100644 --- a/docs/rust-sidecar-standards.zh.md +++ b/docs/rust-sidecar-standards.zh.md @@ -88,6 +88,8 @@ Crate root 和目录级 `mod` glue 可以保留既有的显式本地 prelude, `wdl-rust-common`(`rust/common/`)是本批次唯一的 shared Rust helper crate。它只拥有必须跨 crate 保持一致的小 primitive,例如环境变量数字解析、日志等级解析、HTTP health probe、shutdown/in-flight tracking、通用 JSON log-line emission、wall-clock millisecond helper、短 non-cryptographic random hex suffix、稳定 non-cryptographic hash、queue Redis key builders、worker version / bundle-key parsing、Prometheus metric storage/formatting、Prometheus text response、结构化错误字段合并、internal-auth token/header matching、Redis command construction helper、中立的 Redis connection execution wrapper,以及 UTF-8 安全的字符串截断。它不能变成 service 行为的杂物箱。Axum-facing helper 必须 feature-gate,因此 D1/DO supervisor binaries 这类非 HTTP consumer 不需要付 HTTP stack 成本。 +`test-support` feature 暴露 Rust service tests 共用的唯一 process-environment override helper。它在同一 test process 的 module 之间串行化 override,并在 unwind 时恢复全部值;production dependency build 不启用该 feature。 + 当 sidecar/service 在这些方面行为不同,本地显式代码仍然更好: - service-specific shutdown/drain timing 和 shutdown log event diff --git a/docs/security.zh.md b/docs/security.zh.md index ef90532..21df4f9 100644 --- a/docs/security.zh.md +++ b/docs/security.zh.md @@ -94,7 +94,7 @@ Binding 是 tenant capability 的主要 surface: Terraform 在 ECS Fargate 上运行平台服务,包括执行 tenant 的 runtime task。Tenant-running task 的 cloud credential 暴露由 least-privilege task role、public-only workerd outbound binding 和 private mesh security group 约束。ECS Exec 只应在需要 platform operator access 的地方启用。 -Service Connect 和 security group 是 internal mesh 边界的一部分。共享 `WDL_INTERNAL_AUTH_TOKEN` 当前值必须在 runtime、d1-runtime、do-runtime、scheduler、workflows 和 redis-proxy sidecar 间保持一致;可选 previous 值只作为维护窗口轮换桥接被 receiver 接受。Caller 始终发送当前值,因此 token 轮换不是 rolling-safe,除非先暂停 traffic 或一起重启 private fleet。Scheduler 是 runtime internal dispatch 和 workflows tick 的 client;Workflows 会把 Durable Object alarm dispatch 到 do-runtime。Scheduler 和 workflows 都不是公开 service target。 +Service Connect 和 security group 是 internal mesh 边界的一部分。共享 `WDL_INTERNAL_AUTH_TOKEN` 当前值必须在 runtime、d1-runtime、do-runtime、scheduler、workflows 和 redis-proxy sidecar 间保持一致;可选的 `WDL_INTERNAL_AUTH_PREVIOUS_TOKEN` 只作为维护窗口轮换桥接被 receiver 接受。Caller 始终发送当前值,因此 token 轮换不是 rolling-safe,除非先暂停 traffic 或一起重启 private fleet。Scheduler 是 runtime internal dispatch 和 workflows tick 的 client;Workflows 会把 Durable Object alarm dispatch 到 do-runtime。Scheduler 和 workflows 都不是公开 service target。 ## Tenant-facing 合同 diff --git a/docs/source-map.md b/docs/source-map.md index f8b3c9b..a95880a 100644 --- a/docs/source-map.md +++ b/docs/source-map.md @@ -36,8 +36,10 @@ are outside this map unless they own runtime or deployable service behavior. | `runtime/lib.js` | Pure runtime helpers such as bundle-to-worker-code, byte normalization, and dispatch body normalization. | | `control/index.js` | Thin HTTP dispatcher on system-runtime `:8082`; delegates to handlers after auth. | | `control/handlers/` | Endpoint handlers for deploy, promote, versions, workers, delete, secrets, hosts, reload, auth tokens, D1, R2, workflows, and log tail. | -| `control/shared.js` | Control singletons, auth wrapper, JSON/error helpers, Redis publish helpers, and shared lifecycle/delete helpers. Direct `state.*` access belongs here or in the dispatcher. | -| `control/lib.js` | Control route-to-action classifier, route utilities, delete-lock key helper, and referrer redaction. | +| `control/shared.js` | Control singletons, auth wrapper, Redis publish helpers, state-bound workflow transport wiring, and shared lifecycle/delete helpers. Direct `state.*` access belongs here or in the dispatcher. | +| `control/errors.js`, `control/json-body.js`, `control/optimistic.js` | Pure Control error-response and bounded JSON request-body contracts plus the strict `WatchError`/Redis-session adapter over the shared optimistic retry loop, re-exported by `control/shared.js`. | +| `control/workflows-client.js` | Internal Control-to-Workflows POST transport with explicit caller-owned timeout selection; callers own endpoint-specific response interpretation. | +| `control/lib.js` | Pure Control data-shaping: route-to-action classification, key helpers, canonical bundle `__meta__` parsing, and referrer redaction. | | `control/bundle.js` | Bundle/module normalization, compatibility metadata, vars, and emitted module manifest construction. | | `control/bindings.js` | Service/platform binding parsers, ACL evaluation, and linker helpers. | | `control/topology.js` | Route, pattern, cron, queue consumer, and workflow declaration parsing for deploy metadata. | @@ -56,23 +58,27 @@ are outside this map unless they own runtime or deployable service behavior. | Path | Responsibility | |---|---| | `shared/redis.js`, `shared/redis-*.js` | Public Redis import surface plus split RESP codec, per-call client, WATCH/MULTI session, and subscriber loop modules. Runtime hot paths prefer the Rust redis-proxy sidecar. | -| `shared/owner-lease.js`, `shared/owner-protocol.js`, `shared/owner-forwarder.js` | Shared owner lease parsing, generation counters, key derivation, fence matching, staged Redis owner writes, and owner-forwarding HTTP mechanics used by D1 and DO runtimes. | +| `shared/redis-lock.js` | Token-fenced Redis lock creation, acquire, renewal, and best-effort token-scoped release shared by Control and Auth. | +| `shared/optimistic-retry.js` | Generic bounded optimistic retry loop used by Control, Auth, and the D1/DO owner-lease adapters. | +| `shared/owner-endpoint.js`, `shared/owner-lease.js`, `shared/owner-protocol.js`, `shared/owner-forwarder.js` | Shared owner endpoint grammar, lease parsing, generation counters, key derivation, fence matching, staged Redis owner writes, and authenticated forwarding mechanics used by Control and the D1/DO runtimes. | | `shared/auth-roles.js` | Role table, principal validation, reserved namespace policy, and auth action capabilities. | | `shared/auth-token.js` | Shared `x-admin-token` sanitizer used by control and auth. | | `shared/internal-auth.js` | Shared internal mesh auth header and token helpers used by JS callers and receivers. | | `shared/secret-envelope.js`, `shared/secret-keys.js` | Secret envelope encryption/decryption, canonical base64/JSON handling, AAD binding helpers, and secret Redis key construction. | +| `shared/base64.js` | Dependency-free byte/text base64 codec shared by workerd tiers, with a `Buffer` fast path under `nodejs_compat`. | | `shared/hex.js`, `shared/random-id.js`, `shared/errors.js` | Small dependency-free primitives for byte-to-hex rendering, random hex ids, and string-only error message extraction. | | `shared/observability.js` | Structured logger, metrics registry, request-id helpers, and log-level handling for JS tiers. | | `shared/respond.js` | Shared HTTP response, JSON error, Prometheus text, best-effort response body discard, and `x-request-id` echo helpers. | -| `shared/bounded-body.js` | Shared bounded request body byte/text readers; each tier maps limit errors to its own HTTP error contract. | -| `shared/ns-pattern.js` | Namespace, worker, binding, queue, KV id, module path, reserved object-key, and reserved namespace grammars. | -| `shared/version.js` | Worker version formatting and bundle key helpers. | -| `shared/workerd-compat-flags.js` | Pinned mirror of upstream workerd experimental compatibility enable flags used to reject tenant metadata before cold-load. | +| `shared/bounded-body.js` | Shared bounded byte-stream and request-body readers; each tier maps limit errors to its own contract. | +| `shared/ns-pattern.js` | Platform-domain normalization plus namespace, worker, binding, queue, KV/D1/R2 id, module path, reserved object-key, and reserved namespace grammars. | +| `shared/version.js` | Worker version formatting plus worker, route-plane, lifecycle, DO owner-scope key, and route-invalidation channel helpers. | +| `shared/workerd-compat-flags.js` | Pinned upstream mirror of experimental enable flags plus WDL-owned dynamic-worker date and error-serialization policy. | | `shared/queue-keys.js` | JavaScript queue key helpers used by tests and cross-tier key-shape checks. | | `shared/route-projection.js` | Compact pattern-route projection encoding shared by control writers, delete checks, and gateway readers. | | `shared/d1-*.js`, `shared/sql-splitter.js` | D1 parameter, data-field, transport, timeout, query-wire, and SQL splitting utilities shared by runtime, d1-runtime, control, and tests. | | `shared/fnv1a32.js` | Shared JavaScript FNV-1a helpers for runtime-side shard and slot hashing. | | `shared/s3-query.js` | S3 query encoder used by the s3-cleanup system worker; runtime R2 keeps the same standalone helper in `runtime/r2-utils.js` because that file is injected as worker source. | +| `shared/s3-retry.js` | Bounded transient retry policy for idempotent S3 POST operations, shared by runtime R2 and the s3-cleanup worker. | | `shared/s3-xml.js` | Shared S3 XML parsing helpers used by control R2, runtime R2, and system cleanup paths. | | `shared/worker-id.js` | Shared `x-worker-id` formatting, parsing, and runtime-load identity grammar used by gateway, runtime, DO runtime, and tests. | | `shared/cron-time.js` | Control-side cron parsing and slot-alignment helpers; scheduler advancement uses Rust `croner`. | @@ -106,7 +112,7 @@ are outside this map unless they own runtime or deployable service behavior. | `examples/` | Manual demos and reference projects. Tests should not silently depend on them unless the fixture graduates to `test-workers/`. | | `scripts/run-integration-tests.js` | Integration worker-pool runner. | | `scripts/compile-workerd-configs.js` | Compiles workerd Cap'n Proto configs into `dist/workerd-configs/*.bin`. | -| `scripts/extract-workerd-experimental-compat-flags.mjs` | Pin-bump flag extractor. | +| `scripts/extract-workerd-experimental-compat-flags.mjs` | Pin-bump experimental-flag extractor. | ## Infrastructure diff --git a/docs/source-map.zh.md b/docs/source-map.zh.md index 88eece2..6a7d92e 100644 --- a/docs/source-map.zh.md +++ b/docs/source-map.zh.md @@ -33,8 +33,10 @@ | `runtime/lib.js` | 纯 runtime helpers,例如 bundle-to-worker-code、byte normalization 和 dispatch body normalization。 | | `control/index.js` | system-runtime `:8082` 上的薄 HTTP dispatcher;auth 后交给 handlers。 | | `control/handlers/` | deploy、promote、versions、workers、delete、secrets、hosts、reload、auth tokens、D1、R2、workflows 和 log tail endpoint handlers。 | -| `control/shared.js` | Control singletons、auth wrapper、JSON/error helpers、Redis publish helpers 和共享 lifecycle/delete helpers。Direct `state.*` access 只应在这里或 dispatcher。 | -| `control/lib.js` | Control route-to-action classifier、route utilities、delete-lock key helper 和 referrer redaction。 | +| `control/shared.js` | Control singletons、auth wrapper、Redis publish helpers、state-bound workflow transport wiring 和共享 lifecycle/delete helpers。Direct `state.*` access 只应在这里或 dispatcher。 | +| `control/errors.js`、`control/json-body.js`、`control/optimistic.js` | 由 `control/shared.js` re-export 的纯 Control error-response、bounded JSON request-body contract,以及 shared optimistic retry loop 上的 strict `WatchError`/Redis-session adapter。 | +| `control/workflows-client.js` | timeout 由 caller 显式选择的 Control-to-Workflows internal POST transport;endpoint-specific response interpretation 仍由 caller 持有。 | +| `control/lib.js` | 纯 Control data-shaping:route-to-action classification、key helpers、canonical bundle `__meta__` parsing 和 referrer redaction。 | | `control/bundle.js` | Bundle/module normalization、compatibility metadata、vars 和 emitted module manifest construction。 | | `control/bindings.js` | Service/platform binding parsers、ACL evaluation 和 linker helpers。 | | `control/topology.js` | Deploy metadata 中 routes、patterns、cron、queue consumer 和 workflow declaration parsing。 | @@ -53,23 +55,27 @@ | Path | 责任 | |---|---| | `shared/redis.js`、`shared/redis-*.js` | 公共 Redis import surface,以及拆分后的 RESP codec、per-call client、WATCH/MULTI session 和 subscriber loop modules。Runtime hot path 优先使用 Rust redis-proxy sidecar。 | -| `shared/owner-lease.js`、`shared/owner-protocol.js`、`shared/owner-forwarder.js` | D1 和 DO runtimes 共用的 owner lease parsing、generation counters、key derivation、fence matching、staged Redis owner writes 和 owner-forwarding HTTP mechanics。 | +| `shared/redis-lock.js` | Control 和 Auth 共用的 token-fenced Redis lock creation、acquire、renewal 和 best-effort token-scoped release。 | +| `shared/optimistic-retry.js` | Control、Auth 和 D1/DO owner-lease adapter 共用的通用有界 optimistic retry loop。 | +| `shared/owner-endpoint.js`、`shared/owner-lease.js`、`shared/owner-protocol.js`、`shared/owner-forwarder.js` | Control 与 D1/DO runtimes 共用的 owner endpoint grammar、owner lease parsing、generation counters、key derivation、fence matching、staged Redis owner writes 和 authenticated forwarding mechanics。 | | `shared/auth-roles.js` | Role table、principal validation、reserved namespace policy 和 auth action capabilities。 | | `shared/auth-token.js` | Control 和 auth 共用的 `x-admin-token` sanitizer。 | | `shared/internal-auth.js` | JS caller 和 receiver 共用的 internal mesh auth header / token helpers。 | | `shared/secret-envelope.js`、`shared/secret-keys.js` | Secret envelope encryption/decryption、canonical base64/JSON handling、AAD binding helpers 和 secret Redis key construction。 | +| `shared/base64.js` | Workerd tiers 共用的无依赖 byte/text base64 codec;在 `nodejs_compat` 下使用 `Buffer` fast path。 | | `shared/hex.js`、`shared/random-id.js`、`shared/errors.js` | byte-to-hex rendering、random hex ids 和 string-only error message extraction 的无依赖小 primitive。 | | `shared/observability.js` | JS tiers 的 structured logger、metrics registry、request-id helpers 和 log-level handling。 | | `shared/respond.js` | 共享 HTTP response、JSON error、Prometheus text、best-effort response body discard 和 `x-request-id` echo helpers。 | -| `shared/bounded-body.js` | 共享 bounded request body byte/text readers;各 tier 自己把 limit error 映射为对应 HTTP error contract。 | -| `shared/ns-pattern.js` | Namespace、worker、binding、queue、KV id、module path、reserved object-key 和 reserved namespace grammars。 | -| `shared/version.js` | Worker version formatting 和 bundle key helpers。 | -| `shared/workerd-compat-flags.js` | 上游 workerd experimental compatibility enable flags 的 pinned mirror,用于在 cold-load 前拒绝 tenant metadata。 | +| `shared/bounded-body.js` | 共享 bounded byte-stream 和 request-body readers;各 tier 自己把 limit error 映射为对应 contract。 | +| `shared/ns-pattern.js` | Platform-domain normalization,以及 namespace、worker、binding、queue、KV/D1/R2 id、module path、reserved object-key 和 reserved namespace grammars。 | +| `shared/version.js` | Worker version formatting,以及 worker、route-plane、lifecycle、DO owner-scope key 与 route invalidation channel helpers。 | +| `shared/workerd-compat-flags.js` | 上游 workerd experimental enable flags 的 pinned mirror,以及 WDL-owned dynamic-worker 日期和 error-serialization policy。 | | `shared/queue-keys.js` | JavaScript queue key helpers,供 tests 和 cross-tier key-shape checks 使用。 | | `shared/route-projection.js` | Control writer、delete check 和 gateway reader 共用的紧凑 pattern-route projection encoding。 | | `shared/d1-*.js`、`shared/sql-splitter.js` | Runtime、d1-runtime、control 和 tests 共用的 D1 parameter、data-field、transport、timeout、query-wire 和 SQL splitting utilities。 | | `shared/fnv1a32.js` | Runtime-side shard 和 slot hashing 共用的 JavaScript FNV-1a helpers。 | | `shared/s3-query.js` | s3-cleanup system worker 使用的 S3 query encoder;runtime R2 在 `runtime/r2-utils.js` 保留同一套 standalone helper,因为该文件会作为 worker source 注入。 | +| `shared/s3-retry.js` | runtime R2 与 s3-cleanup worker 共用的 idempotent S3 POST 有界瞬态重试策略。 | | `shared/s3-xml.js` | Control R2、runtime R2 和 system cleanup 路径共用的 S3 XML parsing helpers。 | | `shared/worker-id.js` | Gateway、runtime、DO runtime 和 tests 共用的 `x-worker-id` formatting、parsing 和 runtime-load identity grammar。 | | `shared/cron-time.js` | Control 侧 cron parsing 和 slot-alignment helpers;scheduler advancement 使用 Rust `croner`。 | @@ -103,7 +109,7 @@ | `examples/` | 手工 demo 和 reference projects。测试不应悄悄依赖它们,除非 fixture 明确迁入 `test-workers/`。 | | `scripts/run-integration-tests.js` | Integration worker-pool runner。 | | `scripts/compile-workerd-configs.js` | 把 workerd Cap'n Proto configs 编译成 `dist/workerd-configs/*.bin`。 | -| `scripts/extract-workerd-experimental-compat-flags.mjs` | pin bump flag 提取脚本。 | +| `scripts/extract-workerd-experimental-compat-flags.mjs` | pin bump experimental flag 提取脚本。 | ## Infrastructure diff --git a/docs/testing.md b/docs/testing.md index 503e24e..1e0d519 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -201,7 +201,10 @@ Prefer the narrow helper that matches the response or fixture source: static-analysis utilities (`source-scan.js`), and `mocks/` for mocked Cloudflare runtime surfaces. `load-shared-module.js` owns repository module source loading, data-URL construction, and import-specifier rewrite helpers; - use it instead of ad hoc `readFileSync(...).replace(...)` chains when a test + load import-free contract owners such as `shared/version.js` and + `shared/ns-pattern.js` directly with `repositoryFileUrl(...)` instead of + reimplementing their grammar in a stub. + Use it instead of ad hoc `readFileSync(...).replace(...)` chains when a test rewrites repo module imports. Use `readRepositoryFile(...)` plus `applyModuleReplacements(...)` for local source rewrites, `readRepositoryModuleSource(...)` when reading repository module source and @@ -216,7 +219,9 @@ Prefer the narrow helper that matches the response or fixture source: `control/handlers/*` unit tests. Use it for state/env/log/metrics/backend injection instead of declaring another file-local `control-shared` data-URL stub when the handler under test follows the shared control entrypoint - shape. + shape. The harness stub imports pure response, bounded JSON body, coded abort, + and optimistic retry contracts from their production owners; keep only + state-bound wiring and test instrumentation local to the stub. `load-runtime-r2-binding.js` owns the `runtime/bindings/r2.js` host binding module graph, including SigV4Client/fetch recording hooks. Extend that loader for additional R2 host tests instead of copying the replacement graph back @@ -227,7 +232,10 @@ Prefer the narrow helper that matches the response or fixture source: `mocks/fake-redis.js` owns the reusable in-memory Redis session/multi subset for unit tests, including command tracing for reads, writes, batch helpers, and multi ops; extend it when multiple tests need the same Redis command - shape instead of adding another file-local mock. + shape instead of adding another file-local mock. Use + `sharedRedisStubUrl(...)` for data-URL `shared-redis` replacements so tests + share the fake `WatchError` constructor and production `decodeBulk` + semantics; append only graph-specific state or client exports. `request-body.js` owns typed JSON request-body parsing for mock backends; use it when tests capture `RequestInit.body` instead of open-coding `JSON.parse(...)`. `do-envelope.js` owns test decoding for Durable Object diff --git a/docs/testing.zh.md b/docs/testing.zh.md index 516782c..373d98c 100644 --- a/docs/testing.zh.md +++ b/docs/testing.zh.md @@ -164,10 +164,10 @@ npm install -g @wdl-dev/cli@1.4.0 - 被 JavaScript 和 Rust 两侧测试共同读取的 cross-language JSON fixture 放在 `tests/fixtures/`。JavaScript 测试用 `readRepositoryJson(...)`;Rust 测试用 `include_str!(...)` 读取同一个文件。这类 fixture 只 pin 测试合同和 drift guard,不新增 runtime shared owner。 - `tests/helpers/` 是 unit test helper 的归属: - 包含 `load-*.js` ESM data-URL loader 家族(每个被测 repo 模块一份 loader,例如 `load-auth-lib.js`、`load-control-lib.js`、`load-runtime-dispatch.js`)、共享 fixture(`runtime-dispatch-fixtures.js`、`control-shared-stub.js`)、静态分析工具(`source-scan.js`),以及 mock 化 Cloudflare runtime 表面的 `mocks/`。 - - `load-shared-module.js` 负责仓库模块源码读取、data-URL 构造和 import specifier rewrite helper;测试需要重写 repo module import 时,用它代替手写 `readFileSync(...).replace(...)` 链。局部 source rewrite 使用 `readRepositoryFile(...)` 加 `applyModuleReplacements(...)`;需要读取 repo module source 并一次性应用 replacements 时用 `readRepositoryModuleSource(...)`;可复用 repo module 用 `repositoryModuleDataUrl(...)`;import-map 形态的 stub 用 `importSpecifierReplacements(...)`。这是被测试锁住的约定:`tests/unit/test-helper-style-contracts.test.js` 会拒绝共享 loader helper 之外新增的 source-producer `.replace(...)` module rewrite 链,包括直接链、变量中转形态,以及对 `applyModuleReplacements(...)` 产物的二次 rewrite。 - - `control-handler-harness.js` 负责 `control/handlers/*` unit tests 共用的 `control-shared` harness;handler 遵循 shared control entrypoint 形态时,用它注入 state/env/log/metrics/backend service 和 Redis,不再新增 file-local `control-shared` data-URL stub。 + - `load-shared-module.js` 负责仓库模块源码读取、data-URL 构造和 import specifier rewrite helper;`shared/version.js`、`shared/ns-pattern.js` 这类无 import 的 contract owner 应通过 `repositoryFileUrl(...)` 直接加载,不在 stub 中重写其 grammar。测试需要重写 repo module import 时,用它代替手写 `readFileSync(...).replace(...)` 链。局部 source rewrite 使用 `readRepositoryFile(...)` 加 `applyModuleReplacements(...)`;需要读取 repo module source 并一次性应用 replacements 时用 `readRepositoryModuleSource(...)`;可复用 repo module 用 `repositoryModuleDataUrl(...)`;import-map 形态的 stub 用 `importSpecifierReplacements(...)`。这是被测试锁住的约定:`tests/unit/test-helper-style-contracts.test.js` 会拒绝共享 loader helper 之外新增的 source-producer `.replace(...)` module rewrite 链,包括直接链、变量中转形态,以及对 `applyModuleReplacements(...)` 产物的二次 rewrite。 + - `control-handler-harness.js` 负责 `control/handlers/*` unit tests 共用的 `control-shared` harness;handler 遵循 shared control entrypoint 形态时,用它注入 state/env/log/metrics/backend service 和 Redis,不再新增 file-local `control-shared` data-URL stub。Harness stub 从 production owner 导入纯 response、bounded JSON body、coded abort 和 optimistic retry contract;本地只保留 state-bound wiring 和 test instrumentation。 - `load-runtime-r2-binding.js` 负责 `runtime/bindings/r2.js` host binding module graph,包括 SigV4Client/fetch 记录钩子;新增 R2 host 测试时扩展这个 loader,不把 replacement graph 复制回测试文件。`load-auth-index.js` 负责 auth entrypoint mock state accessor;测试使用 `authMockState(...)`、`authLogs(...)` 和 `lastAuthLog(...)`,不直接触碰 `globalThis.__authMockState`。 - - `mocks/fake-redis.js` 负责 unit tests 可复用的 in-memory Redis session/multi 子集,并为读写、batch helper 和 multi ops 提供 command trace;多个测试需要同一 Redis command shape 时扩展它,不再新增 file-local mock。 + - `mocks/fake-redis.js` 负责 unit tests 可复用的 in-memory Redis session/multi 子集,并为读写、batch helper 和 multi ops 提供 command trace;多个测试需要同一 Redis command shape 时扩展它,不再新增 file-local mock。Data-URL `shared-redis` replacement 使用 `sharedRedisStubUrl(...)`,共享 fake `WatchError` constructor 和 production `decodeBulk` semantics;只追加 graph-specific state 或 client exports。 - `request-body.js` 负责 mock backend 捕获 `RequestInit.body` 时的 typed JSON request-body 解析;这类测试不要再直接手写 `JSON.parse(...)`。`do-envelope.js` 负责 Durable Object binary invoke envelope 的测试解码,不再新增 file-local `decodeDoEnvelope()`。style-contract 套件会拒绝新的直接 mock request-body `JSON.parse(...)` 和 file-local DO envelope decoder。 - `mock-global.js` 负责临时替换 global 和 global property;单个 lexical async scope 用 `withMockedGlobal(...)` / `withMockedProperty(...)`,只有文件需要 before/after hook 所有权时才用 install-style helper。`mock-fetch.js` 是这套 helper 的 typed fetch wrapper。style-contract 套件会扫描仓库作者维护的 `.js` 测试,并拒绝这些 helper 之外新增的直接非 `__` `globalThis. = ...` 赋值、直接 `console.log/warn/error/info = ...` 赋值,以及测试常用 built-in/global property hook(`process.stderr.*`、`AbortSignal.*`、`Object.*`、`Headers.prototype.*`、`Array.prototype.*`)的直接赋值。 - unit tests 不起 Docker,通过这些 loader 导入。 diff --git a/gateway/index.js b/gateway/index.js index c9c90a8..cc89f53 100644 --- a/gateway/index.js +++ b/gateway/index.js @@ -19,6 +19,7 @@ import { createHttpRequestScope, } from "shared-request-scope"; import { + deleteGatewayInternalHeaders, isWebSocketUpgrade, normalizeRequestHost, } from "gateway-lib"; @@ -37,6 +38,7 @@ import { runtimeForwardOutcome, } from "gateway-runtime"; import { formatWorkerId } from "shared-worker-id"; +import { platformDomainFromEnv } from "shared-ns-pattern"; // Re-exported so workerd's capnp `durableObjectNamespaces` entry resolves // the class name against this worker's exports. @@ -62,19 +64,6 @@ function notFoundResponse() { } const bindLogLevel = createLogLevelBinder(); -const INTERNAL_HEADER_PREFIX = "x-wdl-"; -const INTERNAL_FORWARD_HEADERS = [ - "x-worker-id", - "x-worker-prefix", -]; - -/** @param {Headers} headers */ -function stripClientInternalHeaders(headers) { - for (const header of INTERNAL_FORWARD_HEADERS) headers.delete(header); - for (const header of [...headers.keys()]) { - if (header.toLowerCase().startsWith(INTERNAL_HEADER_PREFIX)) headers.delete(header); - } -} export default { /** @@ -86,7 +75,6 @@ export default { bindLogLevel(env); const url = new URL(request.url); const redis = createGatewayRedis(env.REDIS_ADDR); - const platformDomain = normalizeRequestHost(env.PLATFORM_DOMAIN || "workers.local").toLowerCase(); /** @type {string | null} */ let namespace = null; /** @type {string | null} */ @@ -102,10 +90,11 @@ export default { extras: () => ({ namespace, worker, version }), }); - const subscriberStart = ensureGatewaySubscriber(env.REDIS_ADDR); - if (subscriberStart) ctx.waitUntil(subscriberStart); - try { + const platformDomain = platformDomainFromEnv(env); + const subscriberStart = ensureGatewaySubscriber(env.REDIS_ADDR); + if (subscriberStart) ctx.waitUntil(subscriberStart); + if (url.pathname === "/healthz" && request.method === "GET") { scope.setRoute("healthz"); return scope.respond(jsonResponse(200, { @@ -145,7 +134,7 @@ export default { url.pathname = dispatch.forwardPath; const forwardRequest = new Request(url.toString(), request); - stripClientInternalHeaders(forwardRequest.headers); + deleteGatewayInternalHeaders(forwardRequest.headers); // Loader branches carry worker identity + prefix; control is // infrastructure and has no worker id to inject. if (dispatch.bindingName !== "CONTROL") { diff --git a/gateway/lib.js b/gateway/lib.js index e083977..25aa0ec 100644 --- a/gateway/lib.js +++ b/gateway/lib.js @@ -8,6 +8,20 @@ * @typedef {{ slot: string, reason: string }} PatternError */ +const INTERNAL_HEADER_PREFIX = "x-wdl-"; +const INTERNAL_FORWARD_HEADERS = [ + "x-worker-id", + "x-worker-prefix", +]; + +/** @param {Headers} headers */ +export function deleteGatewayInternalHeaders(headers) { + for (const name of INTERNAL_FORWARD_HEADERS) headers.delete(name); + for (const name of [...headers.keys()]) { + if (name.toLowerCase().startsWith(INTERNAL_HEADER_PREFIX)) headers.delete(name); + } +} + /** @param {string} s */ export function escapeRegex(s) { return RegExp.escape(s); diff --git a/gateway/runtime.js b/gateway/runtime.js index fcb847c..7804994 100644 --- a/gateway/runtime.js +++ b/gateway/runtime.js @@ -11,7 +11,15 @@ import { recordRedisCommand, } from "shared-observability"; import { isValidRouteNs } from "shared-ns-pattern"; -import { patternsKey, routesKey } from "shared-version"; +import { + DECLARED_HOSTS_KEY, + NAMESPACES_KEY, + PATTERNS_CHANNEL, + ROUTES_CHANNEL, + ROUTES_FLUSH_CHANNEL, + patternsKey, + routesKey, +} from "shared-version"; import { isCanonicalPatternHost, sortPatterns } from "gateway-lib"; /** @type {Set | null} */ @@ -28,7 +36,6 @@ let websocketProxyDetachedConnections = 0; let websocketProxyBufferedMessages = 0; const MAX_ROUTE_CACHE_ENTRIES = 10_000; const MAX_PATTERN_CACHE_ENTRIES = 10_000; -const DECLARED_HOSTS_KEY = "declared-hosts"; const utf8Decoder = new TextDecoder(); export const metrics = new MetricsRegistry(); @@ -56,7 +63,7 @@ export function createGatewayRedis(redisAddr) { /** @param {RedisClient} redis */ export async function ensureKnownNs(redis) { - if (knownNs === null) knownNs = new Set(await redis.sMembers("namespaces")); + if (knownNs === null) knownNs = new Set(await redis.sMembers(NAMESPACES_KEY)); return knownNs; } @@ -148,7 +155,7 @@ export function ensureGatewaySubscriber(redisAddr) { if (subscriber) return null; subscriber = new RedisSubscriber( redisAddr, - ["routes:invalidate", "routes:flush", "patterns:invalidate"], + [ROUTES_CHANNEL, ROUTES_FLUSH_CHANNEL, PATTERNS_CHANNEL], { onConnect: () => { subscriberConnected = 1; @@ -172,7 +179,7 @@ export function ensureGatewaySubscriber(redisAddr) { }, onMessage: (channel, payload) => { const value = utf8Decoder.decode(payload); - if (channel === "patterns:invalidate") { + if (channel === PATTERNS_CHANNEL) { if (value === "*") { clearPatternState(); } else if (isCanonicalPatternHost(value)) { @@ -192,14 +199,14 @@ export function ensureGatewaySubscriber(redisAddr) { log("info", "patterns_invalidated", { host: value }); return; } - if (channel === "routes:flush") { + if (channel === ROUTES_FLUSH_CHANNEL) { clearRouteState(); metrics.increment("subscriber_invalidations", { service: "gateway", scope: "all" }); log("info", "routes_invalidated_all", {}); return; } - // routes:invalidate owns routeCache + knownNs only; the patterns cache - // is invalidated exclusively via patterns:invalidate so channel + // The route channel owns routeCache + knownNs only; the patterns cache + // is invalidated exclusively via the pattern channel so channel // semantics stay orthogonal. if (!isValidRouteNs(value)) { log("warn", "routes_invalidation_ignored", { diff --git a/gateway/websocket.js b/gateway/websocket.js index ec9ec12..065042c 100644 --- a/gateway/websocket.js +++ b/gateway/websocket.js @@ -1,5 +1,6 @@ import { logStructured } from "shared-observability"; import { discardResponseBody } from "shared-respond"; +import { deleteGatewayInternalHeaders } from "gateway-lib"; /** * @typedef {{ @@ -41,11 +42,6 @@ function websocketClosedNormally(evt) { const RECONNECT_DELAYS_MS = [0, 100, 250, 500, 1000, 2000, 5000]; const MAX_BUFFERED_CLIENT_MESSAGES = 64; const MAX_BUFFERED_CLIENT_MESSAGES_CAP = 1024; -const INTERNAL_HEADER_PREFIX = "x-wdl-"; -const INTERNAL_FORWARD_HEADERS = [ - "x-worker-id", - "x-worker-prefix", -]; const proxyOptionsByEnv = new WeakMap(); /** @param {unknown} value */ @@ -58,10 +54,7 @@ function parseNonNegativeInteger(value) { /** @param {Headers} headers */ function publicWebSocketResponseHeaders(headers) { const out = new Headers(headers); - for (const name of INTERNAL_FORWARD_HEADERS) out.delete(name); - for (const name of [...out.keys()]) { - if (name.toLowerCase().startsWith(INTERNAL_HEADER_PREFIX)) out.delete(name); - } + deleteGatewayInternalHeaders(out); return out; } diff --git a/knip.jsonc b/knip.jsonc index 5b0f14d..3436be5 100644 --- a/knip.jsonc +++ b/knip.jsonc @@ -9,6 +9,10 @@ "d1-runtime/index.js", "do-runtime/index.js", "gateway/index.js", + // Injected into loaded workers and imported by the generated binding + // wrapper, so this is an external module entry rather than an ignored + // source file. Its static dependencies remain covered by Knip. + "runtime/d1-client.js", "runtime/do-owner-network.js", "runtime/index.js", "runtime/tail-worker.js", @@ -27,14 +31,6 @@ "**/*.mjs", "**/*.cjs" ], - // These sources are embedded as text by workerd capnp and injected into - // loaded workers, so no ordinary JS import points at the files. - "ignoreFiles": [ - "runtime/_wdl-d1-params.js", - "runtime/_wdl-d1-transport.js", - "runtime/_wdl-sql-splitter.js", - "runtime/d1-client.js" - ], "ignore": [ "shared/vendor/**", "examples/**", diff --git a/package-lock.json b/package-lock.json index fd5ab68..4e696e7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,7 +21,8 @@ "eslint": "^10.6.0", "globals": "^17.7.0", "knip": "^6.25.0", - "typescript": "^6.0.3" + "typescript": "^6.0.3", + "yaml": "^2.9.0" }, "engines": { "node": ">=24" @@ -31,7 +32,7 @@ "license": "Apache-2.0", "author": "Sean Consulting OÜ", "dependencies": { - "@wdl-dev/aws-sigv4": "^2.0.0", + "@wdl-dev/aws-sigv4": "^3.0.1", "croner": "^10.0.1" } }, @@ -1548,9 +1549,9 @@ } }, "node_modules/@wdl-dev/aws-sigv4": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/@wdl-dev/aws-sigv4/-/aws-sigv4-2.0.0.tgz", - "integrity": "sha512-7Fh4/i8OaRs/P4V4sf0z7gFEsvc1joHTQK6thc4VWwmie6OqI6UpJ7wYz8gPfFzBIQYfmtD6WwwcdeybPHotkg==", + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/@wdl-dev/aws-sigv4/-/aws-sigv4-3.0.1.tgz", + "integrity": "sha512-XXdWT8OQt7TeJd8MQ9QXQJGFpMcnE3HS89mGvdi0Oe1Zx7y78mUK27gTYZXkMlbhsAsZsfCeg2XdDN1j2x/ggA==", "license": "Apache-2.0", "engines": { "node": ">=24" diff --git a/package.json b/package.json index 4b1c8cf..09a6760 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ ], "scripts": { "lint": "eslint .", - "lint:unused": "knip --include exports,types,files --no-config-hints", + "lint:unused": "knip --include exports,types,files --no-config-hints --treat-tag-hints-as-errors", "typecheck": "tsc --noEmit", "typecheck:strict": "tsc --project tsconfig.strict.json", "test:unit": "node --test --test-reporter=spec 'tests/unit/**/*.test.js'", @@ -42,7 +42,8 @@ "eslint": "^10.6.0", "globals": "^17.7.0", "knip": "^6.25.0", - "typescript": "^6.0.3" + "typescript": "^6.0.3", + "yaml": "^2.9.0" }, "dependencies": { "workerd": "1.20260701.1" diff --git a/runtime/_wdl-do-transport.js b/runtime/_wdl-do-transport.js index dd3b2a8..1a74d10 100644 --- a/runtime/_wdl-do-transport.js +++ b/runtime/_wdl-do-transport.js @@ -1,4 +1,5 @@ import { validOwnerEndpointForService } from "./_wdl-owner-endpoint.js"; +import { sanitizeRequestId } from "./_wdl-request-id.js"; export const DO_INVOKE_URL = "http://do-runtime/internal/do/invoke"; export const DO_CONNECT_URL = "http://do-runtime/internal/do/connect"; @@ -7,8 +8,10 @@ export const MAX_DO_REQUEST_BODY_BYTES = 1024 * 1024; export const MAX_DO_INVOKE_ENVELOPE_BYTES = 2 * 1024 * 1024; export const MAX_DO_REQUEST_HEADER_COUNT = 128; export const MAX_DO_REQUEST_HEADER_BYTES = 64 * 1024; +const MAX_DO_RPC_METHOD_BYTES = 256; export const DO_ACCEPT_OWNER_HINT_HEADER = "x-wdl-do-accept-owner-hint"; export const DO_OWNER_HINT_CONTROL_HEADER = "x-wdl-do-owner-hint"; +export const DO_OWNERSHIP_ERROR_CONTROL_HEADER = "x-wdl-do-ownership-error"; export const DO_OWNER_HINT_CODE = "do_owner_hint"; export const INTERNAL_AUTH_HEADER = "x-wdl-internal-auth"; @@ -21,6 +24,7 @@ const DO_OWNER_HINT_HEADERS = { const DO_OWNER_HINT_STRIP_HEADERS = [ ...Object.values(DO_OWNER_HINT_HEADERS), DO_OWNER_HINT_CONTROL_HEADER, + DO_OWNERSHIP_ERROR_CONTROL_HEADER, ]; const DO_FETCH_STRIP_HEADERS = [ ...DO_OWNER_HINT_STRIP_HEADERS, @@ -35,13 +39,16 @@ const OWNER_ENDPOINT_UNAVAILABLE_STATUSES = new Set([502, 503, 504]); const OWNER_RACE_RETRY_CODES = new Set([ "stale_owner_generation", "owner_claim_raced", + "owner_fence_missing", "owner_lease_expired", "owner_lease_too_short", ]); const OWNER_HINT_STALE_CODES = new Set([ "stale_owner_generation", "owner_claim_raced", + "owner_fence_missing", "owner_lease_expired", + "stale_owner_storage", "owner_lease_too_short", "owner_renew_raced", "owner_release_raced", @@ -50,6 +57,83 @@ const OWNER_HINT_STALE_CODES = new Set([ "forward_hop_exhausted", "task_draining", ]); +const IntrinsicArray = Array; +const IntrinsicDataView = DataView; +const IntrinsicHeaders = Headers; +const IntrinsicJSON = JSON; +const IntrinsicNumber = Number; +const IntrinsicObject = Object; +const IntrinsicRequest = Request; +const IntrinsicResponse = Response; +const IntrinsicUint8Array = Uint8Array; +const IntrinsicWeakSet = WeakSet; +const intrinsicReflectApply = Reflect.apply; +const intrinsicArrayForEach = Array.prototype.forEach; +const intrinsicArrayIsArray = Array.isArray; +const intrinsicArrayPush = Array.prototype.push; +const intrinsicDataViewSetUint32 = DataView.prototype.setUint32; +const intrinsicHeadersAppend = Headers.prototype.append; +const intrinsicHeadersDelete = Headers.prototype.delete; +const intrinsicHeadersForEach = Headers.prototype.forEach; +const intrinsicHeadersGet = Headers.prototype.get; +const intrinsicHeadersSet = Headers.prototype.set; +const intrinsicJsonStringify = JSON.stringify; +const intrinsicNumberIsFinite = Number.isFinite; +const intrinsicNumberIsSafeInteger = Number.isSafeInteger; +const intrinsicObjectCreate = Object.create; +const intrinsicObjectEntries = Object.entries; +const intrinsicObjectGetOwnPropertySymbols = Object.getOwnPropertySymbols; +const intrinsicObjectGetPrototypeOf = Object.getPrototypeOf; +const intrinsicObjectSetPrototypeOf = Object.setPrototypeOf; +const intrinsicPromiseThen = Promise.prototype.then; +const intrinsicResponseJson = Response.json; +const intrinsicReadableStreamCancel = ReadableStream.prototype.cancel; +const intrinsicReadableStreamGetReader = ReadableStream.prototype.getReader; +const intrinsicReadableStreamReaderCancel = ReadableStreamDefaultReader.prototype.cancel; +const intrinsicReadableStreamReaderRead = ReadableStreamDefaultReader.prototype.read; +const intrinsicReadableStreamReaderReleaseLock = ReadableStreamDefaultReader.prototype.releaseLock; +const intrinsicRegExpTest = RegExp.prototype.test; +const intrinsicSetHas = Set.prototype.has; +const intrinsicStringToLowerCase = String.prototype.toLowerCase; +const intrinsicStringToUpperCase = String.prototype.toUpperCase; +const intrinsicTextEncoderEncode = TextEncoder.prototype.encode; +const intrinsicUint8ArraySet = Uint8Array.prototype.set; +const intrinsicUint8ArrayBufferGet = /** @type {(this: Uint8Array) => ArrayBufferLike} */ ( + prototypeGetter(Uint8Array.prototype, "buffer") +); +const intrinsicUint8ArrayByteLengthGet = /** @type {(this: Uint8Array) => number} */ ( + prototypeGetter(Uint8Array.prototype, "byteLength") +); +const intrinsicWeakSetAdd = WeakSet.prototype.add; +const intrinsicWeakSetDelete = WeakSet.prototype.delete; +const intrinsicWeakSetHas = WeakSet.prototype.has; +const intrinsicRequestHeadersGet = /** @type {(this: Request) => Headers} */ ( + prototypeGetter(Request.prototype, "headers") +); +const intrinsicRequestBodyGet = /** @type {(this: Request) => ReadableStream | null} */ ( + prototypeGetter(Request.prototype, "body") +); +const intrinsicRequestMethodGet = /** @type {(this: Request) => string} */ ( + prototypeGetter(Request.prototype, "method") +); +const intrinsicRequestUrlGet = /** @type {(this: Request) => string} */ ( + prototypeGetter(Request.prototype, "url") +); +const intrinsicResponseBodyGet = /** @type {(this: Response) => ReadableStream | null} */ ( + prototypeGetter(Response.prototype, "body") +); +const intrinsicResponseHeadersGet = /** @type {(this: Response) => Headers} */ ( + prototypeGetter(Response.prototype, "headers") +); +const intrinsicResponseStatusGet = /** @type {(this: Response) => number} */ ( + prototypeGetter(Response.prototype, "status") +); +const intrinsicResponseStatusTextGet = /** @type {(this: Response) => string} */ ( + prototypeGetter(Response.prototype, "statusText") +); +const intrinsicResponseWebSocketGet = /** @type {((this: Response) => WebSocket | null) | undefined} */ ( + prototypeGetter(Response.prototype, "webSocket") +); const utf8Encoder = new TextEncoder(); const DO_CONNECT_HEADERS = { @@ -67,27 +151,271 @@ const RPC_RESERVED_METHODS = new Set(["fetch", "alarm"]); * @typedef {{ ns: string, worker: string, version: string, doStorageId: string, className: string }} DoBindingProps * @typedef {{ ownerKey: string, taskId: string, endpoint: string, generation: number }} DoOwnerHint * @typedef {(url: string, init?: RequestInit) => Promise} DoFetch + * @typedef {{ + * get: (value: unknown) => unknown, + * set: (value: unknown, hint: unknown) => unknown, + * delete: (value: unknown) => unknown, + * }} DoOwnerHintCache + * @typedef {{ + * routerFetch: DoFetch, + * routerUrl: string, + * ownerFetch: DoFetch | typeof fetch | null | undefined, + * ownerPath: string, + * init: RequestInit, + * cache: DoOwnerHintCache, + * hintKey: string, + * replayOwnerUnavailable?: boolean, + * }} DoOwnerHintDispatchOptions */ +// workerd places some WebIDL mixin getters on a parent prototype. Resolve +// them once before tenant module evaluation rather than reading patched getters. +/** @param {object | null} prototype @param {string} name */ +function prototypeGetter(prototype, name) { + let current = prototype; + while (current) { + const getter = Object.getOwnPropertyDescriptor(current, name)?.get; + if (getter) return getter; + current = Object.getPrototypeOf(current); + } + return undefined; +} + /** @param {DoBindingProps} props @param {string} objectName */ export function doOwnerHintCacheKey(props, objectName) { return `${props.doStorageId}:${props.className}:${objectName}`; } /** @param {Response} response */ -export async function staleDoOwnerHintResponse(response) { - if (response.status < 400) return false; +export function staleDoOwnerHintResponse(response) { + if (responseStatus(response) < 400) return false; + const code = responseHeader(response, DO_OWNERSHIP_ERROR_CONTROL_HEADER); + return code !== null && setHas(OWNER_HINT_STALE_CODES, code); +} + +/** @template T @param {T[]} values @param {(value: T) => void} callback */ +function forEachArray(values, callback) { + intrinsicReflectApply(intrinsicArrayForEach, values, [callback]); +} + +/** @template T @param {T[]} values @param {T} value */ +function pushArray(values, value) { + intrinsicReflectApply(intrinsicArrayPush, values, [value]); +} + +/** @param {Uint8Array} target @param {Uint8Array} source @param {number} offset */ +function setBytes(target, source, offset) { + intrinsicReflectApply(intrinsicUint8ArraySet, target, [source, offset]); +} + +/** @param {Uint8Array} value */ +function byteArrayBuffer(value) { + return intrinsicReflectApply(intrinsicUint8ArrayBufferGet, value, []); +} + +/** @param {Uint8Array} value */ +function byteArrayLength(value) { + return intrinsicReflectApply(intrinsicUint8ArrayByteLengthGet, value, []); +} + +/** @param {Headers} headers @param {string} name @param {string} value */ +function headerAppend(headers, name, value) { + intrinsicReflectApply(intrinsicHeadersAppend, headers, [name, value]); +} + +/** @param {Headers} headers @param {string} name */ +function headerDelete(headers, name) { + intrinsicReflectApply(intrinsicHeadersDelete, headers, [name]); +} + +/** @param {Headers} headers @param {string} name */ +function headerValue(headers, name) { + return intrinsicReflectApply(intrinsicHeadersGet, headers, [name]); +} + +/** @param {Headers} headers @param {string} name */ +/** @param {Headers} headers @param {string} name @param {string} value */ +function headerSet(headers, name, value) { + intrinsicReflectApply(intrinsicHeadersSet, headers, [name, value]); +} + +/** @param {Headers} source */ +function copyHeaders(source) { + const out = new IntrinsicHeaders(); + intrinsicReflectApply(intrinsicHeadersForEach, source, [ + /** @param {string} value @param {string} name */ + (value, name) => headerAppend(out, name, value), + ]); + return out; +} + +/** @param {Headers} headers */ +function headerEntries(headers) { + /** @type {[string, string][]} */ + const out = []; + intrinsicReflectApply(intrinsicHeadersForEach, headers, [ + /** @param {string} value @param {string} name */ + (value, name) => pushArray(out, [name, value]), + ]); + return out; +} + +/** @template T @param {Set} set @param {T} value */ +function setHas(set, value) { + return intrinsicReflectApply(intrinsicSetHas, set, [value]); +} + +/** @param {string} value */ +function stringToLowerCase(value) { + return intrinsicReflectApply(intrinsicStringToLowerCase, value, []); +} + +/** @param {string} value */ +function stringToUpperCase(value) { + return intrinsicReflectApply(intrinsicStringToUpperCase, value, []); +} + +/** @param {RegExp} regexp @param {string} value */ +function regexpTest(regexp, value) { + return intrinsicReflectApply(intrinsicRegExpTest, regexp, [value]); +} + +/** @param {Request} request */ +function requestHeaders(request) { + return intrinsicReflectApply(intrinsicRequestHeadersGet, request, []); +} + +/** @param {Request} request */ +function requestBody(request) { + return intrinsicReflectApply(intrinsicRequestBodyGet, request, []); +} + +/** @param {Request} request */ +function requestMethod(request) { + return intrinsicReflectApply(intrinsicRequestMethodGet, request, []); +} + +/** @param {Request} request */ +function requestUrl(request) { + return intrinsicReflectApply(intrinsicRequestUrlGet, request, []); +} + +/** @param {ReadableStream} stream */ +function streamReader(stream) { + return /** @type {ReadableStreamDefaultReader} */ ( + intrinsicReflectApply(intrinsicReadableStreamGetReader, stream, []) + ); +} + +/** @param {ReadableStreamDefaultReader} reader */ +function readStreamChunk(reader) { + return intrinsicReflectApply(intrinsicReadableStreamReaderRead, reader, []); +} + +/** @param {ReadableStreamDefaultReader} reader */ +function releaseStreamReader(reader) { + intrinsicReflectApply(intrinsicReadableStreamReaderReleaseLock, reader, []); +} + +/** @param {ReadableStreamDefaultReader} reader */ +function cancelStreamReader(reader) { + try { + const promise = intrinsicReflectApply(intrinsicReadableStreamReaderCancel, reader, []); + intrinsicReflectApply(intrinsicPromiseThen, promise, [undefined, () => {}]); + } catch { + // Cancellation is best-effort after the bounded reader has already rejected. + } +} + +/** @param {Response} response */ +function cancelResponseBody(response) { + // This source is injected into loaded workers, so keep cleanup local rather + // than adding shared-respond as another injected facade module. try { - const body = await response.clone().json(); - return OWNER_HINT_STALE_CODES.has(body?.error); + const body = responseBody(response); + if (!body) return; + const promise = intrinsicReflectApply(intrinsicReadableStreamCancel, body, []); + intrinsicReflectApply(intrinsicPromiseThen, promise, [undefined, () => {}]); } catch { - return false; + // Best-effort cleanup only; the replacement response owns behavior. } } +/** @param {Response} response */ +function responseBody(response) { + return intrinsicReflectApply(intrinsicResponseBodyGet, response, []); +} + +/** @param {Response} response */ +function responseHeaders(response) { + return intrinsicReflectApply(intrinsicResponseHeadersGet, response, []); +} + +/** @param {Response} response */ +function responseStatus(response) { + return intrinsicReflectApply(intrinsicResponseStatusGet, response, []); +} + +/** @param {Response} response */ +function responseStatusText(response) { + return intrinsicReflectApply(intrinsicResponseStatusTextGet, response, []); +} + +/** @param {Response} response */ +function responseWebSocket(response) { + if (!intrinsicResponseWebSocketGet) return null; + return intrinsicReflectApply(intrinsicResponseWebSocketGet, response, []); +} + +/** @param {Response} response @param {string} name */ +function responseHeader(response, name) { + return headerValue(responseHeaders(response), name); +} + /** @param {string} value */ function encodeUtf8(value) { - return utf8Encoder.encode(value); + return intrinsicReflectApply(intrinsicTextEncoderEncode, utf8Encoder, [value]); +} + +/** @param {unknown} value */ +function numberValue(value) { + return intrinsicReflectApply(IntrinsicNumber, undefined, [value]); +} + +/** @param {unknown} value */ +function stringifyJson(value) { + return intrinsicReflectApply(intrinsicJsonStringify, IntrinsicJSON, [cloneJsonData(value)]); +} + +/** @param {unknown} value @param {WeakSet} [seen] */ +function cloneJsonData(value, seen = new IntrinsicWeakSet()) { + if (value === null || typeof value !== "object") return value; + const objectValue = /** @type {object} */ (value); + if (intrinsicReflectApply(intrinsicWeakSetHas, seen, [objectValue])) { + throw new TypeError("Cannot serialize circular JSON data"); + } + intrinsicReflectApply(intrinsicWeakSetAdd, seen, [objectValue]); + try { + if (intrinsicReflectApply(intrinsicArrayIsArray, IntrinsicArray, [objectValue])) { + const source = /** @type {unknown[]} */ (objectValue); + const out = new IntrinsicArray(source.length); + intrinsicReflectApply(intrinsicObjectSetPrototypeOf, IntrinsicObject, [out, null]); + for (let i = 0; i < source.length; i++) out[i] = cloneJsonData(source[i], seen); + return out; + } + const out = /** @type {Record} */ ( + intrinsicReflectApply(intrinsicObjectCreate, IntrinsicObject, [null]) + ); + const entries = /** @type {[string, unknown][]} */ ( + intrinsicReflectApply(intrinsicObjectEntries, IntrinsicObject, [objectValue]) + ); + forEachArray(entries, (entry) => { + out[entry[0]] = cloneJsonData(entry[1], seen); + }); + return out; + } finally { + intrinsicReflectApply(intrinsicWeakSetDelete, seen, [objectValue]); + } } /** @@ -96,21 +424,25 @@ function encodeUtf8(value) { * @returns {Uint8Array} */ function encodeDoInvokeEnvelope(metadata, bodyBytes = null) { - const metadataBytes = encodeUtf8(JSON.stringify(metadata)); - const body = bodyBytes == null ? new Uint8Array() : bodyBytes; - if (metadataBytes.length + body.length + 4 > MAX_DO_INVOKE_ENVELOPE_BYTES) { + const metadataBytes = encodeUtf8(stringifyJson(metadata)); + const body = bodyBytes == null ? new IntrinsicUint8Array() : bodyBytes; + const metadataLength = byteArrayLength(metadataBytes); + const bodyLength = byteArrayLength(body); + const envelopeLength = 4 + metadataLength + bodyLength; + if (envelopeLength > MAX_DO_INVOKE_ENVELOPE_BYTES) { throw new TypeError(`Durable Object invoke envelope exceeds ${MAX_DO_INVOKE_ENVELOPE_BYTES} bytes`); } - const envelope = new Uint8Array(4 + metadataBytes.length + body.length); - new DataView(envelope.buffer, envelope.byteOffset, envelope.byteLength).setUint32(0, metadataBytes.length, false); - envelope.set(metadataBytes, 4); - envelope.set(body, 4 + metadataBytes.length); + const envelope = new IntrinsicUint8Array(envelopeLength); + const view = new IntrinsicDataView(byteArrayBuffer(envelope), 0, envelopeLength); + intrinsicReflectApply(intrinsicDataViewSetUint32, view, [0, metadataLength, false]); + setBytes(envelope, metadataBytes, 4); + setBytes(envelope, body, 4 + metadataLength); return envelope; } /** @param {string} value */ function byteLength(value) { - return utf8Encoder.encode(value).byteLength; + return byteArrayLength(encodeUtf8(value)); } /** @@ -120,40 +452,41 @@ function byteLength(value) { async function readRequestBodyBytes(request) { // This source is injected into loaded workers, so keep the bounded reader // local instead of importing a helper that would add another facade module. - const contentLength = request.headers.get("content-length"); + const contentLength = headerValue(requestHeaders(request), "content-length"); if (contentLength != null && contentLength !== "") { - const declared = Number(contentLength); - if (Number.isFinite(declared) && declared > MAX_DO_REQUEST_BODY_BYTES) { + const declared = numberValue(contentLength); + if (intrinsicReflectApply(intrinsicNumberIsFinite, IntrinsicNumber, [declared]) && declared > MAX_DO_REQUEST_BODY_BYTES) { throw new TypeError(`Durable Object fetch body exceeds ${MAX_DO_REQUEST_BODY_BYTES} bytes`); } } - if (!request.body) return new Uint8Array(); + const requestStream = requestBody(request); + if (!requestStream) return new IntrinsicUint8Array(); - const reader = request.body.getReader(); + const reader = streamReader(requestStream); /** @type {Uint8Array[]} */ const chunks = []; let total = 0; try { while (true) { - const { done, value } = await reader.read(); + const { done, value } = await readStreamChunk(reader); if (done) break; - total += value.byteLength; + total += byteArrayLength(value); if (total > MAX_DO_REQUEST_BODY_BYTES) { - reader.cancel().catch(() => {}); + cancelStreamReader(reader); throw new TypeError(`Durable Object fetch body exceeds ${MAX_DO_REQUEST_BODY_BYTES} bytes`); } - chunks.push(value); + pushArray(chunks, value); } } finally { - reader.releaseLock(); + releaseStreamReader(reader); } - const body = new Uint8Array(total); + const body = new IntrinsicUint8Array(total); let offset = 0; - for (const chunk of chunks) { - body.set(chunk, offset); - offset += chunk.byteLength; - } + forEachArray(chunks, (chunk) => { + setBytes(body, chunk, offset); + offset += byteArrayLength(chunk); + }); return body; } @@ -161,53 +494,71 @@ async function readRequestBodyBytes(request) { * @param {unknown} value * @param {string} field * @param {WeakSet} [seen] - * @returns {string | null} + * @returns {unknown} */ -function jsonDataError(value, field, seen = new WeakSet()) { +function cloneJsonRpcData(value, field, seen = new IntrinsicWeakSet()) { if (value === null) return null; const type = typeof value; - if (type === "string" || type === "boolean") return null; - if (type === "number") return Number.isFinite(value) ? null : `${field} must be a finite number`; + if (type === "string" || type === "boolean") return value; + if (type === "number") { + if (!intrinsicReflectApply(intrinsicNumberIsFinite, IntrinsicNumber, [value])) { + throw new TypeError(`${field} must be a finite number`); + } + return value; + } if (type === "bigint" || type === "function" || type === "symbol" || type === "undefined") { - return `${field} must be JSON data`; + throw new TypeError(`${field} must be JSON data`); } - if (type !== "object") return `${field} must be JSON data`; + if (type !== "object") throw new TypeError(`${field} must be JSON data`); const objectValue = /** @type {Record | unknown[]} */ (value); - if (seen.has(objectValue)) return `${field} must not be circular`; - seen.add(objectValue); - if (Array.isArray(objectValue)) { - for (let i = 0; i < objectValue.length; i++) { - if (!(i in objectValue)) return `${field} must not be sparse`; - const error = jsonDataError(objectValue[i], `${field}[${i}]`, seen); - if (error) return error; - } - seen.delete(objectValue); - return null; + if (intrinsicReflectApply(intrinsicWeakSetHas, seen, [objectValue])) { + throw new TypeError(`${field} must not be circular`); } - const proto = Object.getPrototypeOf(objectValue); - if (proto !== Object.prototype && proto !== null) return `${field} must be a plain JSON object`; - if (Object.getOwnPropertySymbols(objectValue).length > 0) return `${field} must not contain symbol keys`; - for (const [key, entry] of Object.entries(objectValue)) { - const error = jsonDataError(entry, `${field}.${key}`, seen); - if (error) return error; + intrinsicReflectApply(intrinsicWeakSetAdd, seen, [objectValue]); + try { + if (intrinsicReflectApply(intrinsicArrayIsArray, IntrinsicArray, [objectValue])) { + const arrayValue = /** @type {unknown[]} */ (objectValue); + const length = arrayValue.length; + const out = new IntrinsicArray(length); + intrinsicReflectApply(intrinsicObjectSetPrototypeOf, IntrinsicObject, [out, null]); + for (let i = 0; i < length; i++) { + if (!(i in arrayValue)) throw new TypeError(`${field} must not be sparse`); + out[i] = cloneJsonRpcData(arrayValue[i], `${field}[${i}]`, seen); + } + return out; + } + const proto = intrinsicReflectApply(intrinsicObjectGetPrototypeOf, IntrinsicObject, [objectValue]); + if (proto !== IntrinsicObject.prototype && proto !== null) { + throw new TypeError(`${field} must be a plain JSON object`); + } + const symbols = intrinsicReflectApply(intrinsicObjectGetOwnPropertySymbols, IntrinsicObject, [objectValue]); + if (symbols.length > 0) throw new TypeError(`${field} must not contain symbol keys`); + const out = /** @type {Record} */ ( + intrinsicReflectApply(intrinsicObjectCreate, IntrinsicObject, [null]) + ); + const entries = /** @type {[string, unknown][]} */ ( + intrinsicReflectApply(intrinsicObjectEntries, IntrinsicObject, [objectValue]) + ); + for (let i = 0; i < entries.length; i++) { + const entry = entries[i]; + out[entry[0]] = cloneJsonRpcData(entry[1], `${field}.${entry[0]}`, seen); + } + return out; + } finally { + intrinsicReflectApply(intrinsicWeakSetDelete, seen, [objectValue]); } - seen.delete(objectValue); - return null; -} - -/** @param {unknown} args */ -export function assertJsonRpcArgs(args) { - if (!Array.isArray(args)) throw new TypeError("rpc.args must be an array"); - const error = jsonDataError(args, "rpc.args"); - if (error) throw new TypeError(error); } /** @param {unknown} method */ export function assertRpcMethod(method) { - if (typeof method !== "string" || !/^[A-Za-z_$][A-Za-z0-9_$]*$/.test(method)) { + if ( + typeof method !== "string" || + byteLength(method) > MAX_DO_RPC_METHOD_BYTES || + !regexpTest(/^[A-Za-z_$][A-Za-z0-9_$]*$/, method) + ) { throw new TypeError("rpc.method is not valid"); } - if (RPC_RESERVED_METHODS.has(method)) { + if (setHas(RPC_RESERVED_METHODS, method)) { throw new TypeError("rpc.method is reserved"); } } @@ -221,8 +572,11 @@ export function assertRpcMethod(method) { */ export function rpcInvokeBody(props, objectName, method, args) { assertRpcMethod(method); - assertJsonRpcArgs(args); - if (byteLength(JSON.stringify(args)) > MAX_DO_REQUEST_BODY_BYTES) { + if (!intrinsicReflectApply(intrinsicArrayIsArray, IntrinsicArray, [args])) { + throw new TypeError("rpc.args must be an array"); + } + const stableArgs = /** @type {unknown[]} */ (cloneJsonRpcData(args, "rpc.args")); + if (byteLength(stringifyJson(stableArgs)) > MAX_DO_REQUEST_BODY_BYTES) { throw new TypeError(`rpc.args exceeds ${MAX_DO_REQUEST_BODY_BYTES} bytes`); } return encodeDoInvokeEnvelope({ @@ -233,7 +587,7 @@ export function rpcInvokeBody(props, objectName, method, args) { className: props.className, objectName, kind: "rpc", - rpc: { method, args }, + rpc: { method, args: stableArgs }, }); } @@ -293,7 +647,7 @@ export function rpcInvokeInit(props, objectName, method, args, requestId) { export async function rpcResultFromResponse(response) { const body = await response.json().catch(() => null); if (!response.ok) { - const err = new Error(body?.message || `Durable Object RPC failed with status ${response.status}`); + const err = new Error(body?.message || `Durable Object RPC failed with status ${responseStatus(response)}`); if (body?.name) err.name = body.name; if (body?.error) Object.defineProperty(err, "code", { value: body.error }); if (typeof body?.stack === "string" && body.stack) err.stack = body.stack; @@ -303,14 +657,10 @@ export async function rpcResultFromResponse(response) { } /** @param {Response} response */ -export async function retryableOwnerRaceResponse(response) { - if (response.status !== 503) return false; - try { - const body = await response.clone().json(); - return OWNER_RACE_RETRY_CODES.has(body?.error); - } catch { - return false; - } +export function retryableOwnerRaceResponse(response) { + if (responseStatus(response) !== 503) return false; + const code = responseHeader(response, DO_OWNERSHIP_ERROR_CONTROL_HEADER); + return code !== null && setHas(OWNER_RACE_RETRY_CODES, code); } /** @@ -319,25 +669,26 @@ export async function retryableOwnerRaceResponse(response) { * @returns {Promise<{ spec: { method: string, url: string, headers: [string, string][] }, bodyBytes: Uint8Array | null }>} */ export async function requestSpec(request, requestId) { - const forwarded = new Request(request); - for (const name of DO_FETCH_STRIP_HEADERS) { - forwarded.headers.delete(name); - } - if (requestId && !forwarded.headers.has("x-request-id")) { - forwarded.headers.set("x-request-id", requestId); - } - const method = forwarded.method.toUpperCase(); - const headers = [...forwarded.headers.entries()]; + const forwarded = new IntrinsicRequest(request); + const forwardedHeaders = requestHeaders(forwarded); + forEachArray(DO_FETCH_STRIP_HEADERS, (name) => { + headerDelete(forwardedHeaders, name); + }); + headerDelete(forwardedHeaders, "x-request-id"); + const canonicalRequestId = sanitizeRequestId(requestId); + if (canonicalRequestId) headerSet(forwardedHeaders, "x-request-id", canonicalRequestId); + const method = stringToUpperCase(requestMethod(forwarded)); + const headers = headerEntries(forwardedHeaders); enforceRequestHeadersBudget(headers); const spec = { method, - url: forwarded.url, + url: requestUrl(forwarded), headers, }; let bodyBytes = null; if (method !== "GET" && method !== "HEAD") { const body = await readRequestBodyBytes(forwarded); - if (body.byteLength > 0) bodyBytes = body; + if (byteArrayLength(body) > 0) bodyBytes = body; } return { spec, bodyBytes }; } @@ -348,17 +699,25 @@ function enforceRequestHeadersBudget(headers) { throw new TypeError(`Durable Object fetch headers exceed ${MAX_DO_REQUEST_HEADER_COUNT} entries`); } let total = 0; - for (const [name, value] of headers) { + forEachArray(headers, (entry) => { + const name = entry[0]; + const value = entry[1]; total += byteLength(name) + byteLength(value); if (total > MAX_DO_REQUEST_HEADER_BYTES) { throw new TypeError(`Durable Object fetch headers exceed ${MAX_DO_REQUEST_HEADER_BYTES} bytes`); } - } + }); } /** @param {Request} request */ export function isWebSocketUpgrade(request) { - return (request.headers.get("Upgrade") || "").toLowerCase() === "websocket"; + return stringToLowerCase(headerValue(requestHeaders(request), "Upgrade") || "") === "websocket"; +} + +/** @param {Request} request */ +export function replayOwnerUnavailableForFetch(request) { + const method = stringToUpperCase(requestMethod(request)); + return method === "GET" || method === "HEAD"; } /** @@ -368,17 +727,19 @@ export function isWebSocketUpgrade(request) { * @param {string | null | undefined} requestId */ export function connectHeaders(props, objectName, request, requestId) { - const headers = new Headers(request.headers); - for (const name of DO_CONNECT_STRIP_HEADERS) headers.delete(name); - headers.set(DO_CONNECT_HEADERS.ns, props.ns); - headers.set(DO_CONNECT_HEADERS.worker, props.worker); - headers.set(DO_CONNECT_HEADERS.version, props.version); - headers.set(DO_CONNECT_HEADERS.doStorageId, props.doStorageId); - headers.set(DO_CONNECT_HEADERS.className, props.className); - headers.set(DO_CONNECT_HEADERS.objectName, objectName); - headers.set(DO_CONNECT_HEADERS.requestUrl, request.url); - if (requestId && !headers.has("x-request-id")) headers.set("x-request-id", requestId); - headers.set(DO_ACCEPT_OWNER_HINT_HEADER, "1"); + const headers = copyHeaders(requestHeaders(request)); + forEachArray(DO_CONNECT_STRIP_HEADERS, (name) => headerDelete(headers, name)); + headerSet(headers, DO_CONNECT_HEADERS.ns, props.ns); + headerSet(headers, DO_CONNECT_HEADERS.worker, props.worker); + headerSet(headers, DO_CONNECT_HEADERS.version, props.version); + headerSet(headers, DO_CONNECT_HEADERS.doStorageId, props.doStorageId); + headerSet(headers, DO_CONNECT_HEADERS.className, props.className); + headerSet(headers, DO_CONNECT_HEADERS.objectName, objectName); + headerSet(headers, DO_CONNECT_HEADERS.requestUrl, requestUrl(request)); + headerDelete(headers, "x-request-id"); + const canonicalRequestId = sanitizeRequestId(requestId); + if (canonicalRequestId) headerSet(headers, "x-request-id", canonicalRequestId); + headerSet(headers, DO_ACCEPT_OWNER_HINT_HEADER, "1"); return headers; } @@ -386,7 +747,7 @@ export function connectHeaders(props, objectName, request, requestId) { * @param {DoOwnerHint} owner * @param {string} pathname */ -export function ownerRequestUrl(owner, pathname) { +function ownerRequestUrl(owner, pathname) { if (!validOwnerEndpoint(owner.endpoint)) { throw new Error("Invalid DO owner endpoint"); } @@ -414,25 +775,29 @@ export function ownerHintHeaders(owner, { control = false } = {}) { * @returns {DoOwnerHint | null} */ export function ownerHintFromHeaders(headers) { - const ownerKey = headers.get(DO_OWNER_HINT_HEADERS.ownerKey); - const taskId = headers.get(DO_OWNER_HINT_HEADERS.taskId); - const endpoint = headers.get(DO_OWNER_HINT_HEADERS.endpoint); - const rawGeneration = headers.get(DO_OWNER_HINT_HEADERS.generation); + const ownerKey = headerValue(headers, DO_OWNER_HINT_HEADERS.ownerKey); + const taskId = headerValue(headers, DO_OWNER_HINT_HEADERS.taskId); + const endpoint = headerValue(headers, DO_OWNER_HINT_HEADERS.endpoint); + const rawGeneration = headerValue(headers, DO_OWNER_HINT_HEADERS.generation); if (rawGeneration == null || rawGeneration === "") return null; - const generation = Number(rawGeneration); - if (!ownerKey || !taskId || !validOwnerEndpoint(endpoint) || !Number.isInteger(generation) || generation < 0) { + const generation = numberValue(rawGeneration); + if ( + !ownerKey || !taskId || !validOwnerEndpoint(endpoint) || + !intrinsicReflectApply(intrinsicNumberIsSafeInteger, IntrinsicNumber, [generation]) || generation <= 0 + ) { return null; } - return { ownerKey, taskId, endpoint: String(endpoint), generation }; + return { ownerKey, taskId, endpoint: /** @type {string} */ (endpoint), generation }; } /** @param {Response} response */ -export async function ownerHintFromResponse(response) { - if (response.status !== 409) return null; +function ownerHintFromResponse(response) { + if (responseStatus(response) !== 409) return null; // Only do-runtime-authored headers are trusted. Tenant DO code controls the // response body and may intentionally return a do_owner_hint-shaped 409. - if (response.headers.get(DO_OWNER_HINT_CONTROL_HEADER) !== "1") return null; - return ownerHintFromHeaders(response.headers); + const headers = responseHeaders(response); + if (headerValue(headers, DO_OWNER_HINT_CONTROL_HEADER) !== "1") return null; + return ownerHintFromHeaders(headers); } /** @param {unknown} endpoint */ @@ -440,135 +805,148 @@ function validOwnerEndpoint(endpoint) { return validOwnerEndpointForService(endpoint, 8788, "do-runtime"); } -/** - * @param {Response} response - * @param {DoFetch | typeof fetch | null | undefined} ownerFetch - * @param {string} pathname - * @param {RequestInit} init - */ -export async function followOwnerHint(response, ownerFetch, pathname, init) { - const hint = await ownerHintFromResponse(response); - if (!hint || typeof ownerFetch !== "function") return response; - const direct = await ownerFetch(ownerRequestUrl(hint, pathname), init); - if (ownerEndpointUnavailableResponse(direct)) { - throw new Error(`DO owner endpoint returned ${direct.status}`); - } - return direct; -} - -export function ownerUnavailableResponse() { - return Response.json( +function ownerUnavailableResponse() { + return intrinsicReflectApply(intrinsicResponseJson, IntrinsicResponse, [ { error: "owner_unavailable", message: "DO owner is unavailable; request outcome may be unknown" }, { status: 503 } - ); + ]); } /** - * @param {{ - * routerFetch: DoFetch, - * routerUrl: string, - * ownerFetch: DoFetch | typeof fetch | null | undefined, - * ownerPath: string, - * init: RequestInit, - * cachedHint?: DoOwnerHint | null, - * rememberHint?: (hint: DoOwnerHint) => void, - * clearHint?: () => void, - * staleCachedResponse?: (response: Response) => Promise, - * bypassOwnerHintResponse?: (response: Response) => Promise, - * replayOwnerUnavailable?: boolean, - * }} options + * @param {DoOwnerHintDispatchOptions} options + * @param {boolean} retryOwnerRace */ -export async function dispatchDoRequestWithOwnerHint({ +async function dispatchDoWithHintCache({ routerFetch, routerUrl, ownerFetch, ownerPath, init, - cachedHint = null, - rememberHint = (_hint) => {}, - clearHint = () => {}, - staleCachedResponse = async (_response) => false, - bypassOwnerHintResponse = async (_response) => false, + cache, + hintKey, replayOwnerUnavailable = false, -}) { +}, retryOwnerRace) { + /** @param {DoOwnerHint} hint */ + const rememberHint = (hint) => cache.set(hintKey, hint); + const clearHint = () => cache.delete(hintKey); + const replayOrUnavailable = async () => replayOwnerUnavailable + ? await routerFetch(routerUrl, withoutOwnerHintOptIn(init)) + : ownerUnavailableResponse(); + /** @param {Response} response */ + const finish = async (response) => { + let result = response; + if (retryOwnerRace && retryableOwnerRaceResponse(result)) { + cancelResponseBody(result); + clearHint(); + result = await routerFetch(routerUrl, withoutOwnerHintOptIn(init)); + } + return stripOwnerHintHeaders(result); + }; + + const cachedHint = /** @type {DoOwnerHint | null} */ (cache.get(hintKey)); if (cachedHint && typeof ownerFetch === "function") { + /** @type {Response} */ + let direct; try { - const direct = await ownerFetch(ownerRequestUrl(cachedHint, ownerPath), init); - if (await ownerHintFromResponse(direct)) { - clearHint(); - } else if (await staleCachedResponse(direct)) { - clearHint(); - } else if (ownerEndpointUnavailableResponse(direct)) { - clearHint(); - if (replayOwnerUnavailable) { - return await routerFetch(routerUrl, withoutOwnerHintOptIn(init)); - } - return ownerUnavailableResponse(); - } else { - const learned = ownerHintFromHeaders(direct.headers); - if (learned) rememberHint(learned); - return direct; - } + direct = await ownerFetch(ownerRequestUrl(cachedHint, ownerPath), init); } catch { clearHint(); - if (replayOwnerUnavailable) { - return await routerFetch(routerUrl, withoutOwnerHintOptIn(init)); - } - return ownerUnavailableResponse(); + return await finish(await replayOrUnavailable()); + } + if (ownerHintFromResponse(direct) || staleDoOwnerHintResponse(direct)) { + cancelResponseBody(direct); + clearHint(); + } else if (ownerEndpointUnavailableResponse(direct)) { + cancelResponseBody(direct); + clearHint(); + return await finish(await replayOrUnavailable()); + } else { + const learned = ownerHintFromHeaders(responseHeaders(direct)); + if (learned) rememberHint(learned); + return await finish(direct); } } const routed = await routerFetch(routerUrl, init); - if (await bypassOwnerHintResponse(routed)) return routed; - const hinted = await ownerHintFromResponse(routed); + if (retryOwnerRace && retryableOwnerRaceResponse(routed)) { + return await finish(routed); + } + + const hinted = ownerHintFromResponse(routed); if (!hinted || typeof ownerFetch !== "function") { if (hinted) rememberHint(hinted); else { - const learned = ownerHintFromHeaders(routed.headers); + const learned = ownerHintFromHeaders(responseHeaders(routed)); if (learned) rememberHint(learned); } - return routed; + return await finish(routed); } + rememberHint(hinted); + cancelResponseBody(routed); + /** @type {Response} */ + let direct; try { - const direct = await followOwnerHint(routed, ownerFetch, ownerPath, init); - const learned = ownerHintFromHeaders(direct.headers); - if (learned) rememberHint(learned); - return direct; + direct = await ownerFetch(ownerRequestUrl(hinted, ownerPath), init); + if (ownerEndpointUnavailableResponse(direct)) { + cancelResponseBody(direct); + throw new Error(`DO owner endpoint returned ${responseStatus(direct)}`); + } } catch { clearHint(); - if (replayOwnerUnavailable) { - return await routerFetch(routerUrl, withoutOwnerHintOptIn(init)); - } - return ownerUnavailableResponse(); + return await finish(await replayOrUnavailable()); + } + if (ownerHintFromResponse(direct)) { + cancelResponseBody(direct); + clearHint(); + return await finish(await replayOrUnavailable()); } + const learned = ownerHintFromHeaders(responseHeaders(direct)); + if (learned) rememberHint(learned); + return await finish(direct); +} + +/** @param {DoOwnerHintDispatchOptions} options */ +export async function dispatchDoInvokeWithHintCache(options) { + return await dispatchDoWithHintCache(options, true); +} + +/** @param {DoOwnerHintDispatchOptions} options */ +export async function dispatchDoConnectWithHintCache(options) { + return await dispatchDoWithHintCache({ + ...options, + replayOwnerUnavailable: false, + }, false); } /** @param {Response} response */ -export function ownerEndpointUnavailableResponse(response) { - return OWNER_ENDPOINT_UNAVAILABLE_STATUSES.has(response.status) && - !ownerHintFromHeaders(response.headers); +function ownerEndpointUnavailableResponse(response) { + return setHas(OWNER_ENDPOINT_UNAVAILABLE_STATUSES, responseStatus(response)) && + !ownerHintFromHeaders(responseHeaders(response)); } /** @param {RequestInit} init */ -export function withoutOwnerHintOptIn(init) { - const headers = new Headers(init.headers || {}); - headers.delete(DO_ACCEPT_OWNER_HINT_HEADER); +function withoutOwnerHintOptIn(init) { + const headers = init.headers instanceof IntrinsicHeaders + ? copyHeaders(init.headers) + : new IntrinsicHeaders(init.headers || {}); + headerDelete(headers, DO_ACCEPT_OWNER_HINT_HEADER); return { ...init, headers }; } /** @param {Response} response */ -export function stripOwnerHintHeaders(response) { - const headers = new Headers(response.headers); - for (const name of DO_OWNER_HINT_STRIP_HEADERS) { - headers.delete(name); - } +function stripOwnerHintHeaders(response) { + const headers = copyHeaders(responseHeaders(response)); + forEachArray(DO_OWNER_HINT_STRIP_HEADERS, (name) => { + headerDelete(headers, name); + }); + const status = responseStatus(response); const init = /** @type {ResponseInit & { webSocket?: WebSocket }} */ ({ - status: response.status, - statusText: response.statusText, + status, + statusText: responseStatusText(response), headers, }); - const webSocket = /** @type {{ webSocket?: WebSocket }} */ (response).webSocket; + const webSocket = responseWebSocket(response); if (webSocket) init.webSocket = webSocket; - return new Response(response.status === 101 ? null : response.body, init); + return new IntrinsicResponse(status === 101 ? null : responseBody(response), init); } diff --git a/runtime/_wdl-owner-endpoint.js b/runtime/_wdl-owner-endpoint.js index 4cfdb54..94625ed 100644 --- a/runtime/_wdl-owner-endpoint.js +++ b/runtime/_wdl-owner-endpoint.js @@ -1,59 +1,2 @@ -const OWNER_ENDPOINT_SERVICE_RE = { - "d1-runtime": /^d1-runtime(?:-[a-z0-9-]+)?$/, - "do-runtime": /^do-runtime(?:-[a-z0-9-]+)?$/, -}; - -const OWNER_ENDPOINT_HEADLESS_RE = { - "d1-runtime": /^d1-runtime-[0-9]+\.d1-runtime-headless(?:\.[a-z0-9-]+\.svc(?:\.cluster\.local)?)?$/, - "do-runtime": /^do-runtime-[0-9]+\.do-runtime-headless(?:\.[a-z0-9-]+\.svc(?:\.cluster\.local)?)?$/, -}; - -/** - * @param {unknown} endpoint - * @param {number} port - * @param {"d1-runtime" | "do-runtime"} serviceName - * @returns {boolean} - */ -export function validOwnerEndpointForService(endpoint, port, serviceName) { - if (typeof endpoint !== "string" || !endpoint) return false; - const serviceRe = OWNER_ENDPOINT_SERVICE_RE[serviceName]; - const headlessRe = OWNER_ENDPOINT_HEADLESS_RE[serviceName]; - if (!serviceRe) throw new Error(`Unknown owner endpoint service ${serviceName}`); - if (/[/?#@[\]\\]/.test(endpoint)) return false; - let url; - try { - url = new URL(`http://${endpoint}`); - } catch { - return false; - } - if (url.host !== endpoint || url.username || url.password || url.pathname !== "/" || url.search || url.hash) { - return false; - } - if (url.port !== String(port)) return false; - // ECS task ENI addresses are VPC-local but not necessarily RFC1918; some - // deployments may use non-RFC1918 VPC-local ranges. Trust comes from - // runtime-authored owner-hint headers, while this check rejects URL injection - // and obviously unsafe endpoint classes. - return serviceRe.test(url.hostname) || headlessRe.test(url.hostname) || acceptableIpv4(url.hostname); -} - -/** - * @param {string} hostname - * @returns {boolean} - */ -function acceptableIpv4(hostname) { - const parts = hostname.split("."); - if (parts.length !== 4) return false; - const octets = parts.map(/** @param {string} part */ (part) => { - if (!/^(?:0|[1-9]\d{0,2})$/.test(part)) return null; - const value = Number(part); - return value >= 0 && value <= 255 ? value : null; - }); - if (octets.some(/** @param {number | null} octet */ (octet) => octet === null)) return false; - const [a, b] = octets; - if (a == null || b == null) return false; - if (a === 0 || a === 127) return false; - if (a === 169 && b === 254) return false; - if (a >= 224) return false; - return true; -} +// Local adapter for the injected DO transport's relative module name. +export { validOwnerEndpointForService } from "../shared/owner-endpoint.js"; diff --git a/runtime/_wdl-owner-hint-cache.js b/runtime/_wdl-owner-hint-cache.js index 33310ce..1595583 100644 --- a/runtime/_wdl-owner-hint-cache.js +++ b/runtime/_wdl-owner-hint-cache.js @@ -1,3 +1,52 @@ +// This module is evaluated before tenant code. Keep cache state inaccessible +// after tenant top-level evaluation mutates the shared isolate realm. +const IntrinsicMap = Map; +const IntrinsicNumber = Number; +const intrinsicReflectApply = Reflect.apply; +const intrinsicMapClear = Map.prototype.clear; +const intrinsicMapDelete = Map.prototype.delete; +const intrinsicMapGet = Map.prototype.get; +const intrinsicMapKeys = Map.prototype.keys; +const intrinsicMapSet = Map.prototype.set; +const intrinsicMapSizeGet = /** @type {(this: Map) => number} */ ( + Object.getOwnPropertyDescriptor(Map.prototype, "size")?.get +); +const intrinsicNumberIsInteger = Number.isInteger; +const intrinsicMapIteratorNext = Object.getPrototypeOf( + intrinsicReflectApply(intrinsicMapKeys, new IntrinsicMap(), []) +).next; + +/** @param {Map} map */ +function mapSize(map) { + return intrinsicReflectApply(intrinsicMapSizeGet, map, []); +} + +/** @param {Map} map @param {unknown} key */ +function mapGet(map, key) { + return intrinsicReflectApply(intrinsicMapGet, map, [key]); +} + +/** @param {Map} map @param {unknown} key @param {unknown} value */ +function mapSet(map, key, value) { + intrinsicReflectApply(intrinsicMapSet, map, [key, value]); +} + +/** @param {Map} map @param {unknown} key */ +function mapDelete(map, key) { + return intrinsicReflectApply(intrinsicMapDelete, map, [key]); +} + +/** @param {Map} map */ +function mapClear(map) { + intrinsicReflectApply(intrinsicMapClear, map, []); +} + +/** @param {Map} map */ +function firstMapKey(map) { + const iterator = intrinsicReflectApply(intrinsicMapKeys, map, []); + return intrinsicReflectApply(intrinsicMapIteratorNext, iterator, []).value; +} + /** * @param {{ * defaultMaxEntries?: number, @@ -9,14 +58,16 @@ export function createOwnerHintCache({ keyFor = (value) => value, } = {}) { /** @type {Map} */ - const entries = new Map(); + const entries = new IntrinsicMap(); /** @type {number | null} */ let maxEntriesForTest = null; function maxEntries() { const override = maxEntriesForTest; - return typeof override === "number" && Number.isInteger(override) && override > 0 - ? Number(override) + return typeof override === "number" && + intrinsicReflectApply(intrinsicNumberIsInteger, IntrinsicNumber, [override]) && + override > 0 + ? override : defaultMaxEntries; } @@ -27,22 +78,22 @@ export function createOwnerHintCache({ } function trim() { - while (entries.size > maxEntries()) { - const oldestKey = entries.keys().next().value; + while (mapSize(entries) > maxEntries()) { + const oldestKey = firstMapKey(entries); if (oldestKey === undefined) break; - entries.delete(oldestKey); + mapDelete(entries, oldestKey); } } return { clearForTest() { - entries.clear(); + mapClear(entries); maxEntriesForTest = null; }, /** @param {number | null} maxEntriesValue */ setMaxEntriesForTest(maxEntriesValue) { - entries.clear(); + mapClear(entries); maxEntriesForTest = maxEntriesValue; }, @@ -50,10 +101,10 @@ export function createOwnerHintCache({ get(value) { const key = keyOf(value); if (key == null) return null; - const hint = entries.get(key); + const hint = mapGet(entries, key); if (!hint) return null; - entries.delete(key); - entries.set(key, hint); + mapDelete(entries, key); + mapSet(entries, key, hint); return hint; }, @@ -64,8 +115,8 @@ export function createOwnerHintCache({ set(value, hint) { const key = keyOf(value); if (key == null) return false; - entries.delete(key); - entries.set(key, hint); + mapDelete(entries, key); + mapSet(entries, key, hint); trim(); return true; }, @@ -74,7 +125,7 @@ export function createOwnerHintCache({ delete(value) { const key = keyOf(value); if (key == null) return false; - return entries.delete(key); + return mapDelete(entries, key); }, }; } diff --git a/runtime/_wdl-request-id.js b/runtime/_wdl-request-id.js index 564bd03..80735d3 100644 --- a/runtime/_wdl-request-id.js +++ b/runtime/_wdl-request-id.js @@ -1,3 +1,37 @@ +const intrinsicArrayIsArray = Array.isArray; +const intrinsicObjectHasOwn = Object.hasOwn; +const intrinsicReflectApply = Reflect.apply; +const intrinsicStringCharCodeAt = String.prototype.charCodeAt; +const intrinsicStringIndexOf = String.prototype.indexOf; +const intrinsicStringSlice = String.prototype.slice; + +/** @param {string} value */ +function trimAsciiWhitespace(value) { + let start = 0; + let end = value.length; + const isWhitespace = (/** @type {number} */ code) => + code === 0x20 || code === 0x09 || code === 0x0a || code === 0x0c || code === 0x0d; + while (start < end && isWhitespace(intrinsicReflectApply(intrinsicStringCharCodeAt, value, [start]))) start += 1; + while (end > start && isWhitespace(intrinsicReflectApply(intrinsicStringCharCodeAt, value, [end - 1]))) end -= 1; + return intrinsicReflectApply(intrinsicStringSlice, value, [start, end]); +} + +/** @param {unknown} raw @returns {string | null} */ +export function sanitizeRequestId(raw) { + if (intrinsicReflectApply(intrinsicArrayIsArray, undefined, [raw])) { + raw = /** @type {unknown[]} */ (raw)[0]; + } + if (typeof raw !== "string") return null; + const comma = intrinsicReflectApply(intrinsicStringIndexOf, raw, [","]); + const first = trimAsciiWhitespace(comma === -1 ? raw : intrinsicReflectApply(intrinsicStringSlice, raw, [0, comma])); + if (!first || first.length > 128) return null; + for (let i = 0; i < first.length; i++) { + const code = intrinsicReflectApply(intrinsicStringCharCodeAt, first, [i]); + if (code < 0x21 || code > 0x7e || code === 0x22 || code === 0x5c) return null; + } + return first; +} + /** * Resolve the request id options shared by loaded-isolate host facades. * @@ -5,17 +39,19 @@ * a stable env wrapper and swaps the current request id through the provider. * * @param {unknown} options - * @param {{ providerFirst?: boolean }} [resolution] * @returns {string | null} */ -export function requestIdFromOptions(options, { providerFirst = true } = {}) { +export function requestIdFromOptions(options) { if (!options || typeof options !== "object") return null; const record = /** @type {{ requestIdProvider?: unknown, requestId?: unknown }} */ (options); const fromProvider = () => { - if (typeof record.requestIdProvider !== "function") return null; - const id = record.requestIdProvider(); - return typeof id === "string" && id ? id : null; + if (!intrinsicReflectApply(intrinsicObjectHasOwn, undefined, [record, "requestIdProvider"])) return null; + const provider = record.requestIdProvider; + if (typeof provider !== "function") return null; + return sanitizeRequestId(intrinsicReflectApply(provider, undefined, [])); }; - const fromValue = () => typeof record.requestId === "string" && record.requestId ? record.requestId : null; - return providerFirst ? fromProvider() || fromValue() : fromValue() || fromProvider(); + const fromValue = () => intrinsicReflectApply(intrinsicObjectHasOwn, undefined, [record, "requestId"]) + ? sanitizeRequestId(record.requestId) + : null; + return fromProvider() || fromValue(); } diff --git a/runtime/bindings/d1.js b/runtime/bindings/d1.js index c921b11..1b99e2e 100644 --- a/runtime/bindings/d1.js +++ b/runtime/bindings/d1.js @@ -19,7 +19,7 @@ import { } from "shared-d1-query-wire"; import { metrics } from "runtime-metrics"; import { createOwnerHintCache } from "runtime-owner-hint-cache"; -import { validOwnerEndpointForService } from "runtime-owner-endpoint"; +import { validOwnerEndpointForService } from "shared-owner-endpoint"; import { serviceNameFromEnv } from "runtime-bindings-proxy"; import { withInternalAuth } from "shared-internal-auth"; import { errorMessage } from "shared-errors"; @@ -33,7 +33,9 @@ const OWNER_HINT_STALE_CODES = new Set([ "not-owner", "owner-not-ready", "owner-unavailable", + "owner-record-invalid", "owner-endpoint-missing", + "owner-endpoint-invalid", "forward-hop-exhausted", "owner-claim-raced", "owner-takeover-raced", @@ -147,8 +149,8 @@ function ownerHintFromHeaders(headers) { if ( !taskId || !validOwnerEndpointForService(endpoint, 8787, "d1-runtime") || - !Number.isInteger(generation) || - generation < 0 + !Number.isSafeInteger(generation) || + generation <= 0 ) { return null; } diff --git a/runtime/bindings/do.js b/runtime/bindings/do.js index ebc63ce..f82ef94 100644 --- a/runtime/bindings/do.js +++ b/runtime/bindings/do.js @@ -4,15 +4,13 @@ import { DO_INVOKE_URL, connectHeaders, doOwnerHintCacheKey, - dispatchDoRequestWithOwnerHint, + dispatchDoConnectWithHintCache, + dispatchDoInvokeWithHintCache, fetchInvokeInit, isWebSocketUpgrade, - retryableOwnerRaceResponse, + replayOwnerUnavailableForFetch, rpcInvokeInit, rpcResultFromResponse, - staleDoOwnerHintResponse, - stripOwnerHintHeaders, - withoutOwnerHintOptIn, } from "runtime-do-transport"; import { createOwnerHintCache } from "runtime-owner-hint-cache"; import { withInternalAuth } from "shared-internal-auth"; @@ -25,12 +23,6 @@ import { withInternalAuth } from "shared-internal-auth"; const ownerHintCache = createOwnerHintCache(); -/** @param {Request} request */ -function replayOwnerUnavailableForFetch(request) { - const method = request.method.toUpperCase(); - return method === "GET" || method === "HEAD"; -} - /** @param {DurableObjectNamespace} binding @returns {DoBinding} */ function doBinding(binding) { return /** @type {DoBinding} */ (/** @type {unknown} */ (binding)); @@ -59,24 +51,16 @@ async function dispatchInvokeWithOwnerHint(binding, init, objectName, { replayOw const backend = doBinding(binding).env.DO_BACKEND; const props = propsOf(binding); const hintKey = doOwnerHintCacheKey(props, objectName); - const response = await dispatchDoRequestWithOwnerHint({ + return await dispatchDoInvokeWithHintCache({ routerFetch: (url, requestInit) => backend.fetch(url, requestInit), routerUrl: DO_INVOKE_URL, ownerFetch: fetch, ownerPath: "/internal/do/invoke", init, - cachedHint: /** @type {import("runtime-do-transport").DoOwnerHint | null} */ (ownerHintCache.get(hintKey)), - rememberHint: (hint) => ownerHintCache.set(hintKey, hint), - clearHint: () => ownerHintCache.delete(hintKey), - staleCachedResponse: staleDoOwnerHintResponse, - bypassOwnerHintResponse: retryableOwnerRaceResponse, + cache: ownerHintCache, + hintKey, replayOwnerUnavailable, }); - if (await retryableOwnerRaceResponse(response)) { - ownerHintCache.delete(hintKey); - return stripOwnerHintHeaders(await backend.fetch(DO_INVOKE_URL, withoutOwnerHintOptIn(init))); - } - return stripOwnerHintHeaders(response); } export class DurableObjectNamespace extends WorkerEntrypoint { @@ -97,18 +81,15 @@ export class DurableObjectNamespace extends WorkerEntrypoint { headers: withInternalAuth(connectHeaders(props, objectName, request, requestId), doBinding(this).env), }; const hintKey = doOwnerHintCacheKey(props, objectName); - return stripOwnerHintHeaders(await dispatchDoRequestWithOwnerHint({ + return await dispatchDoConnectWithHintCache({ routerFetch: (url, requestInit) => doBinding(this).env.DO_BACKEND.fetch(url, requestInit), routerUrl: DO_CONNECT_URL, ownerFetch: fetch, ownerPath: "/internal/do/connect", init, - cachedHint: /** @type {import("runtime-do-transport").DoOwnerHint | null} */ (ownerHintCache.get(hintKey)), - rememberHint: (hint) => ownerHintCache.set(hintKey, hint), - clearHint: () => ownerHintCache.delete(hintKey), - staleCachedResponse: staleDoOwnerHintResponse, - replayOwnerUnavailable: false, - })); + cache: ownerHintCache, + hintKey, + }); } const init = await fetchInvokeInit(props, objectName, request, requestId); init.headers = withInternalAuth(init.headers, doBinding(this).env); diff --git a/runtime/bindings/kv.js b/runtime/bindings/kv.js index 0784e9a..f1bd6a5 100644 --- a/runtime/bindings/kv.js +++ b/runtime/bindings/kv.js @@ -14,8 +14,10 @@ // via JSRPC and execute through the Redis proxy sidecar. import { WorkerEntrypoint } from "cloudflare:workers"; -import { base64ToBytes, bytesToBase64, toBytes } from "runtime-lib"; +import { toBytes } from "runtime-lib"; import { recordBindingOperation } from "runtime-metrics"; +import { base64ToBytes, bytesToBase64 } from "shared-base64"; +import { readBoundedStreamBytes } from "shared-bounded-body"; import { discardResponseBody } from "shared-respond"; import { proxyEndpoint as buildProxyEndpoint, @@ -74,41 +76,6 @@ function stringifyKvMetadata(value) { return json; } -/** - * @param {ReadableStream} stream - * @returns {Promise} - */ -async function readStreamWithLimit(stream) { - const reader = stream.getReader(); - const chunks = []; - let total = 0; - try { - for (;;) { - const { value, done } = await reader.read(); - if (done) break; - const chunk = value instanceof Uint8Array ? value : new Uint8Array(value); - total += chunk.byteLength; - if (total > KV_VALUE_MAX_BYTES) { - try { - await reader.cancel(`KV put: value exceeds ${KV_VALUE_MAX_BYTES} byte limit`); - } catch {} - assertKvValueSize(total); - } - chunks.push(chunk); - } - } finally { - try { reader.releaseLock(); } catch {} - } - if (chunks.length === 1) return chunks[0]; - const out = new Uint8Array(total); - let offset = 0; - for (const chunk of chunks) { - out.set(chunk, offset); - offset += chunk.byteLength; - } - return out; -} - /** * @param {unknown} value * @returns {number} @@ -280,7 +247,11 @@ export class KV extends WorkerEntrypoint { return recordBindingOperation(serviceName(kv), "kv", "put", async () => { let bytes; if (value instanceof ReadableStream) { - bytes = await readStreamWithLimit(value); + bytes = await readBoundedStreamBytes( + value, + KV_VALUE_MAX_BYTES, + () => new TypeError(`KV put: value exceeds ${KV_VALUE_MAX_BYTES} byte limit`) + ); } else { bytes = toBytes(value); assertKvValueSize(bytes.byteLength); diff --git a/runtime/bindings/r2.js b/runtime/bindings/r2.js index 4265f72..98e1d3f 100644 --- a/runtime/bindings/r2.js +++ b/runtime/bindings/r2.js @@ -3,6 +3,8 @@ import { SigV4Client } from "@wdl-dev/aws-sigv4"; import { recordBindingOperation } from "runtime-metrics"; import { serviceNameFromEnv } from "runtime-bindings-proxy"; import { discardResponseBody } from "shared-respond"; +import { bytesToBase64 } from "shared-base64"; +import { S3_TRANSIENT_RETRIES, fetchRetryableS3Post } from "shared-s3-retry"; import { assertR2BufferSize, encodeS3KeyPath, @@ -27,9 +29,6 @@ import { parseListObjects, xmlEscape, xmlUnescape } from "runtime-bindings-r2-xm const DELETE_OBJECTS_BATCH_SIZE = 1000; const LIST_INCLUDE_HEAD_CONCURRENCY = 16; const S3_CLIENT_CACHE_MAX_ENTRIES = 128; -const S3_TRANSIENT_RETRIES = 10; -const S3_RETRY_BASE_MS = 50; -const S3_RETRY_MAX_MS = 5_000; const utf8Encoder = new TextEncoder(); const s3Cache = new Map(); const s3ByBucket = new WeakMap(); @@ -51,8 +50,7 @@ function r2Binding(bucket) { /** @param {BufferSource} bytes */ async function sha256Base64(bytes) { const digest = await crypto.subtle.digest("SHA-256", bytes); - // SHA-256 digests are fixed 32-byte values, so spreading here is bounded. - return btoa(String.fromCharCode(...new Uint8Array(digest))); + return bytesToBase64(new Uint8Array(digest)); } /** @@ -113,47 +111,6 @@ async function mapWithConcurrency(items, concurrency, fn) { return out; } -/** @param {number} ms */ -function delay(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -/** @param {number} attempt */ -function s3RetryDelayMs(attempt) { - return Math.min(S3_RETRY_BASE_MS * 2 ** attempt, S3_RETRY_MAX_MS); -} - -/** @param {Response} response */ -function isTransientS3Response(response) { - return response.status === 429 || response.status >= 500; -} - -/** - * DeleteObjects is a POST, but deleting the same key set is safe to retry. - * Keep this scoped to S3 POST calls with known idempotent semantics. - * @param {{ fetch(url: string, init?: RequestInit): Promise }} client - * @param {string} url - * @param {RequestInit} init - */ -async function fetchRetryableS3Post(client, url, init) { - for (let attempt = 0; attempt <= S3_TRANSIENT_RETRIES; attempt += 1) { - let response; - try { - response = await client.fetch(url, init); - } catch (err) { - if (attempt === S3_TRANSIENT_RETRIES) throw err; - await delay(Math.random() * s3RetryDelayMs(attempt)); - continue; - } - if (attempt === S3_TRANSIENT_RETRIES || !isTransientS3Response(response)) { - return response; - } - await discardResponseBody(response); - await delay(Math.random() * s3RetryDelayMs(attempt)); - } - throw new Error("unreachable S3 retry loop exit"); -} - /** @param {R2BucketBinding} bucket */ function serviceName(bucket) { return serviceNameFromEnv(bucket.env); diff --git a/runtime/bindings/r2/metadata.js b/runtime/bindings/r2/metadata.js index 0094b55..cdcb52f 100644 --- a/runtime/bindings/r2/metadata.js +++ b/runtime/bindings/r2/metadata.js @@ -1,4 +1,9 @@ -import { r2RangeAndSizeFromHeaders } from "runtime-r2-utils"; +import { + R2_HTTP_METADATA_FIELDS, + r2CacheExpiryFromHeaders, + r2RangeAndSizeFromHeaders, + setR2CacheExpiryHeader, +} from "runtime-r2-utils"; /** * @typedef {{ offset?: number, length?: number, suffix?: number, header?: string }} R2RangeOptions @@ -79,22 +84,12 @@ export function headersWithRequestId(requestMeta = {}) { function httpMetadataFromHeaders(headers) { /** @type {Record} */ const out = {}; - const fields = [ - ["content-type", "contentType"], - ["content-language", "contentLanguage"], - ["content-disposition", "contentDisposition"], - ["content-encoding", "contentEncoding"], - ["cache-control", "cacheControl"], - ]; - for (const [header, field] of fields) { + for (const [field, header] of R2_HTTP_METADATA_FIELDS) { const value = headers.get(header); if (value) out[field] = value; } - const expires = headers.get("expires"); - if (expires) { - const ms = new Date(expires).getTime(); - if (Number.isFinite(ms)) out.cacheExpiry = ms; - } + const cacheExpiry = r2CacheExpiryFromHeaders(headers); + if (cacheExpiry != null) out.cacheExpiry = cacheExpiry; return out; } @@ -171,19 +166,10 @@ export function metaFromPutResponse(userKey, headers, bodySize, options = {}) { * @param {R2Options} [httpMetadata] */ export function applyHttpMetadata(headers, httpMetadata = {}) { - const pairs = [ - ["contentType", "content-type"], - ["contentLanguage", "content-language"], - ["contentDisposition", "content-disposition"], - ["contentEncoding", "content-encoding"], - ["cacheControl", "cache-control"], - ]; - for (const [field, header] of pairs) { + for (const [field, header] of R2_HTTP_METADATA_FIELDS) { if (httpMetadata[field]) headers.set(header, String(httpMetadata[field])); } - if (httpMetadata.cacheExpiry != null) { - headers.set("expires", new Date(Number(httpMetadata.cacheExpiry)).toUTCString()); - } + setR2CacheExpiryHeader(headers, httpMetadata.cacheExpiry); } /** diff --git a/runtime/bindings/service.js b/runtime/bindings/service.js index c2c834b..f7b2c5c 100644 --- a/runtime/bindings/service.js +++ b/runtime/bindings/service.js @@ -9,10 +9,10 @@ // entrypoint. Protected by service-binding RPC integration tests. import { WorkerEntrypoint } from "cloudflare:workers"; -import { createLoaderCallback } from "runtime-load"; -import { recordLoadedWorker } from "runtime-state"; -import { emitRuntimeTailEvent, fetchTailFields } from "runtime-tail-forwarder"; +import { getLoadedWorkerStub } from "runtime-load"; +import { fetchTailFields, startTailEnvelope } from "runtime-tail-forwarder"; import { INTERNAL_AUTH_HEADER } from "shared-internal-auth"; +import { sanitizeRequestId } from "shared-observability"; const SERVICE_BINDING_INTERNAL_HEADERS = [ "x-worker-id", @@ -64,7 +64,9 @@ export class ServiceBinding extends WorkerEntrypoint { forwarded.headers.set("x-worker-id", targetId); // ALS doesn't cross JSRPC in workerd — the outer rid isn't visible here, // so we can only preserve what the caller forwarded, never mint one. - const requestId = forwarded.headers.get("x-request-id") || null; + const requestId = sanitizeRequestId(forwarded.headers.get("x-request-id")); + if (requestId) forwarded.headers.set("x-request-id", requestId); + else forwarded.headers.delete("x-request-id"); const identity = { namespace: self.ctx.props.targetNs, workerName: self.ctx.props.targetWorker, @@ -72,49 +74,24 @@ export class ServiceBinding extends WorkerEntrypoint { requestId, }; const baseFields = fetchTailFields(forwarded); - const startedAt = Date.now(); - const startTailEvent = emitRuntimeTailEvent({ + const tail = startTailEnvelope({ env: self.env, ctx: self.ctx, identity, event: "worker_fetch", - phase: "start", fields: baseFields, }); try { const response = await targetStub(self, requestId) .getEntrypoint(targetEntrypoint ?? null, targetEntrypointOpts(self)) .fetch(forwarded); - emitRuntimeTailEvent({ - env: self.env, - ctx: self.ctx, - identity, - event: "worker_fetch", - phase: "finish", - after: startTailEvent, - fields: { - ...baseFields, - outcome: "ok", - status: response.status, - duration_ms: Date.now() - startedAt, - }, + tail.finish({ + outcome: "ok", + status: response.status, }); return response; } catch (err) { - emitRuntimeTailEvent({ - env: self.env, - ctx: self.ctx, - identity, - event: "worker_fetch", - phase: "finish", - after: startTailEvent, - fields: { - ...baseFields, - outcome: "error", - error: err instanceof Error ? err.message : undefined, - duration_ms: Date.now() - startedAt, - }, - }); + tail.finishError(err); throw err; } } @@ -151,7 +128,9 @@ function serviceBinding(binding) { function targetStub(self, requestId) { const { targetNs, targetWorker, targetVersion } = self.ctx.props; const targetId = targetIdOf(self.ctx.props); - const baseCallback = createLoaderCallback({ + // The pinned target version may not be active, so service-binding cold-loads + // record the isolate but deliberately leave sibling eviction disabled. + return getLoadedWorkerStub({ requestId, env: self.env, ctx: self.ctx, @@ -159,14 +138,7 @@ function targetStub(self, requestId) { worker: targetWorker, version: targetVersion, workerId: targetId, - }); - // Record but do not evict — the pinned version may not be the - // active one, so aborting siblings here would thrash gateway traffic. - return self.env.LOADER.get(targetId, async () => { - const code = await baseCallback(); - recordLoadedWorker(targetId); - return code; - }); + }).stub; } /** diff --git a/runtime/config-system.capnp b/runtime/config-system.capnp index 4c54e77..782bc8e 100644 --- a/runtime/config-system.capnp +++ b/runtime/config-system.capnp @@ -66,10 +66,11 @@ const loaderWorker :Workerd.Worker = ( (name = "runtime-bindings-do", esModule = embed "bindings/do.js"), (name = "runtime-bindings-internal-auth-backend", esModule = embed "bindings/internal-auth-backend.js"), (name = "runtime-do-transport", esModule = embed "_wdl-do-transport.js"), - (name = "runtime-owner-endpoint", esModule = embed "_wdl-owner-endpoint.js"), - # _wdl-do-transport.js imports the same helper by relative file path, while - # host bindings import it by bare module name. Workerd needs both entries. - (name = "_wdl-owner-endpoint.js", esModule = embed "_wdl-owner-endpoint.js"), + (name = "_wdl-request-id.js", esModule = embed "_wdl-request-id.js"), + (name = "shared-owner-endpoint", esModule = embed "../shared/owner-endpoint.js"), + # Injected DO transport uses a relative module name; host modules use the + # shared bare name. Both resolve to the same shared contract owner. + (name = "_wdl-owner-endpoint.js", esModule = embed "../shared/owner-endpoint.js"), (name = "runtime-owner-hint-cache", esModule = embed "_wdl-owner-hint-cache.js"), (name = "shared-d1-data-field", esModule = embed "../shared/d1-data-field.js"), (name = "shared-d1-params", esModule = embed "../shared/d1-params.js"), @@ -83,7 +84,7 @@ const loaderWorker :Workerd.Worker = ( (name = "runtime-r2-utils-source", text = embed "r2-utils.js"), (name = "runtime-do-client-source", text = embed "do-client.js"), (name = "runtime-do-transport-source", text = embed "_wdl-do-transport.js"), - (name = "runtime-owner-endpoint-source", text = embed "_wdl-owner-endpoint.js"), + (name = "runtime-owner-endpoint-source", text = embed "../shared/owner-endpoint.js"), (name = "runtime-owner-hint-cache-source", text = embed "_wdl-owner-hint-cache.js"), (name = "runtime-request-id-source", text = embed "_wdl-request-id.js"), (name = "runtime-workflows-client-source", text = embed "workflows-client.js"), @@ -91,11 +92,15 @@ const loaderWorker :Workerd.Worker = ( (name = "hex.js", esModule = embed "../shared/hex.js"), (name = "errors.js", esModule = embed "../shared/errors.js"), (name = "shared-errors", esModule = embed "../shared/errors.js"), + (name = "shared-base64", esModule = embed "../shared/base64.js"), + (name = "shared-version", esModule = embed "../shared/version.js"), (name = "shared-observability", esModule = embed "../shared/observability.js"), (name = "shared-s3-xml", esModule = embed "../shared/s3-xml.js"), (name = "shared-ns-pattern", esModule = embed "../shared/ns-pattern.js"), (name = "shared-workerd-compat-flags", esModule = embed "../shared/workerd-compat-flags.js"), (name = "shared-respond", esModule = embed "../shared/respond.js"), + (name = "shared-s3-retry", esModule = embed "../shared/s3-retry.js"), + (name = "respond.js", esModule = embed "../shared/respond.js"), (name = "shared-bounded-body", esModule = embed "../shared/bounded-body.js"), (name = "shared-request-scope", esModule = embed "../shared/request-scope.js"), (name = "shared-env", esModule = embed "../shared/env.js"), @@ -137,7 +142,7 @@ const loaderWorker :Workerd.Worker = ( const doOwnerNetworkWorker :Workerd.Worker = ( modules = [ (name = "worker", esModule = embed "do-owner-network.js"), - (name = "runtime-owner-endpoint", esModule = embed "_wdl-owner-endpoint.js"), + (name = "shared-owner-endpoint", esModule = embed "../shared/owner-endpoint.js"), (name = "shared-internal-auth", esModule = embed "../shared/internal-auth.js"), ], compatibilityDate = "2026-04-24", @@ -151,10 +156,16 @@ const controlWorker :Workerd.Worker = ( modules = [ (name = "worker", esModule = embed "../control/index.js"), (name = "control-shared", esModule = embed "../control/shared.js"), + (name = "control-errors", esModule = embed "../control/errors.js"), + (name = "control-json-body", esModule = embed "../control/json-body.js"), + (name = "control-optimistic", esModule = embed "../control/optimistic.js"), + (name = "shared-owner-lease", esModule = embed "../shared/owner-lease.js"), + (name = "shared-optimistic-retry", esModule = embed "../shared/optimistic-retry.js"), + (name = "control-workflows-client", esModule = embed "../control/workflows-client.js"), (name = "control-lib", esModule = embed "../control/lib.js"), (name = "control-d1-model", esModule = embed "../control/d1-model.js"), (name = "control-d1-runtime-client", esModule = embed "../control/d1-runtime-client.js"), - (name = "runtime-owner-endpoint", esModule = embed "_wdl-owner-endpoint.js"), + (name = "shared-owner-endpoint", esModule = embed "../shared/owner-endpoint.js"), (name = "control-d1-lifecycle", esModule = embed "../control/d1-lifecycle.js"), (name = "control-d1-migrations", esModule = embed "../control/d1-migrations.js"), (name = "control-d1-store", esModule = embed "../control/d1-store.js"), @@ -162,6 +173,7 @@ const controlWorker :Workerd.Worker = ( (name = "control-bindings", esModule = embed "../control/bindings.js"), (name = "control-topology", esModule = embed "../control/topology.js"), (name = "control-env-budget", esModule = embed "../control/env-budget.js"), + (name = "runtime-load-env-build", esModule = embed "load/env-build.js"), (name = "control-worker-code-budget", esModule = embed "../control/worker-code-budget.js"), (name = "do-runtime-load-code-budget", esModule = embed "../do-runtime/load-code-budget.js"), (name = "control-lifecycle-indexes", esModule = embed "../control/lifecycle-indexes.js"), @@ -195,15 +207,19 @@ const controlWorker :Workerd.Worker = ( (name = "errors.js", esModule = embed "../shared/errors.js"), (name = "shared-hex", esModule = embed "../shared/hex.js"), (name = "shared-random-id", esModule = embed "../shared/random-id.js"), + (name = "shared-redis-lock", esModule = embed "../shared/redis-lock.js"), (name = "shared-errors", esModule = embed "../shared/errors.js"), (name = "base64.js", esModule = embed "../shared/base64.js"), (name = "assets-token.js", esModule = embed "../shared/assets-token.js"), (name = "shared-observability", esModule = embed "../shared/observability.js"), (name = "shared-s3-xml", esModule = embed "../shared/s3-xml.js"), (name = "shared-respond", esModule = embed "../shared/respond.js"), + (name = "shared-s3-retry", esModule = embed "../shared/s3-retry.js"), + (name = "respond.js", esModule = embed "../shared/respond.js"), (name = "shared-bounded-body", esModule = embed "../shared/bounded-body.js"), (name = "shared-request-scope", esModule = embed "../shared/request-scope.js"), (name = "shared-ns-pattern", esModule = embed "../shared/ns-pattern.js"), + (name = "ns-pattern.js", esModule = embed "../shared/ns-pattern.js"), (name = "shared-workerd-compat-flags", esModule = embed "../shared/workerd-compat-flags.js"), (name = "shared-auth-token", esModule = embed "../shared/auth-token.js"), (name = "shared-auth-roles", esModule = embed "../shared/auth-roles.js"), @@ -236,7 +252,7 @@ const controlWorker :Workerd.Worker = ( (name = "runtime-r2-utils-source", text = embed "r2-utils.js"), (name = "runtime-do-client-source", text = embed "do-client.js"), (name = "runtime-do-transport-source", text = embed "_wdl-do-transport.js"), - (name = "runtime-owner-endpoint-source", text = embed "_wdl-owner-endpoint.js"), + (name = "runtime-owner-endpoint-source", text = embed "../shared/owner-endpoint.js"), (name = "runtime-owner-hint-cache-source", text = embed "_wdl-owner-hint-cache.js"), (name = "runtime-request-id-source", text = embed "_wdl-request-id.js"), (name = "runtime-workflows-client-source", text = embed "workflows-client.js"), @@ -300,8 +316,11 @@ const authWorker :Workerd.Worker = ( (name = "shared-ns-pattern", esModule = embed "../shared/ns-pattern.js"), (name = "shared-auth-token", esModule = embed "../shared/auth-token.js"), (name = "shared-auth-roles", esModule = embed "../shared/auth-roles.js"), + (name = "shared-version", esModule = embed "../shared/version.js"), (name = "shared-hex", esModule = embed "../shared/hex.js"), (name = "shared-random-id", esModule = embed "../shared/random-id.js"), + (name = "shared-redis-lock", esModule = embed "../shared/redis-lock.js"), + (name = "shared-optimistic-retry", esModule = embed "../shared/optimistic-retry.js"), ], compatibilityDate = "2026-04-24", compatibilityFlags = ["nodejs_compat"], @@ -321,9 +340,11 @@ const tailWorker :Workerd.Worker = ( modules = [ (name = "worker", esModule = embed "tail-worker.js"), (name = "hex.js", esModule = embed "../shared/hex.js"), + (name = "errors.js", esModule = embed "../shared/errors.js"), (name = "shared-observability", esModule = embed "../shared/observability.js"), (name = "runtime-tail-forwarder", esModule = embed "tail-forwarder.js"), (name = "runtime-bindings-proxy", esModule = embed "bindings/proxy.js"), + (name = "shared-errors", esModule = embed "../shared/errors.js"), (name = "shared-respond", esModule = embed "../shared/respond.js"), (name = "shared-internal-auth", esModule = embed "../shared/internal-auth.js"), ], diff --git a/runtime/config-user.capnp b/runtime/config-user.capnp index d83c9f4..666afce 100644 --- a/runtime/config-user.capnp +++ b/runtime/config-user.capnp @@ -65,10 +65,11 @@ const loaderWorker :Workerd.Worker = ( (name = "runtime-bindings-do", esModule = embed "bindings/do.js"), (name = "runtime-bindings-internal-auth-backend", esModule = embed "bindings/internal-auth-backend.js"), (name = "runtime-do-transport", esModule = embed "_wdl-do-transport.js"), - (name = "runtime-owner-endpoint", esModule = embed "_wdl-owner-endpoint.js"), - # _wdl-do-transport.js imports the same helper by relative file path, while - # host bindings import it by bare module name. Workerd needs both entries. - (name = "_wdl-owner-endpoint.js", esModule = embed "_wdl-owner-endpoint.js"), + (name = "_wdl-request-id.js", esModule = embed "_wdl-request-id.js"), + (name = "shared-owner-endpoint", esModule = embed "../shared/owner-endpoint.js"), + # Injected DO transport uses a relative module name; host modules use the + # shared bare name. Both resolve to the same shared contract owner. + (name = "_wdl-owner-endpoint.js", esModule = embed "../shared/owner-endpoint.js"), (name = "runtime-owner-hint-cache", esModule = embed "_wdl-owner-hint-cache.js"), (name = "shared-d1-data-field", esModule = embed "../shared/d1-data-field.js"), (name = "shared-d1-params", esModule = embed "../shared/d1-params.js"), @@ -82,7 +83,7 @@ const loaderWorker :Workerd.Worker = ( (name = "runtime-r2-utils-source", text = embed "r2-utils.js"), (name = "runtime-do-client-source", text = embed "do-client.js"), (name = "runtime-do-transport-source", text = embed "_wdl-do-transport.js"), - (name = "runtime-owner-endpoint-source", text = embed "_wdl-owner-endpoint.js"), + (name = "runtime-owner-endpoint-source", text = embed "../shared/owner-endpoint.js"), (name = "runtime-owner-hint-cache-source", text = embed "_wdl-owner-hint-cache.js"), (name = "runtime-request-id-source", text = embed "_wdl-request-id.js"), (name = "runtime-workflows-client-source", text = embed "workflows-client.js"), @@ -90,11 +91,15 @@ const loaderWorker :Workerd.Worker = ( (name = "hex.js", esModule = embed "../shared/hex.js"), (name = "errors.js", esModule = embed "../shared/errors.js"), (name = "shared-errors", esModule = embed "../shared/errors.js"), + (name = "shared-base64", esModule = embed "../shared/base64.js"), + (name = "shared-version", esModule = embed "../shared/version.js"), (name = "shared-observability", esModule = embed "../shared/observability.js"), (name = "shared-s3-xml", esModule = embed "../shared/s3-xml.js"), (name = "shared-ns-pattern", esModule = embed "../shared/ns-pattern.js"), (name = "shared-workerd-compat-flags", esModule = embed "../shared/workerd-compat-flags.js"), (name = "shared-respond", esModule = embed "../shared/respond.js"), + (name = "shared-s3-retry", esModule = embed "../shared/s3-retry.js"), + (name = "respond.js", esModule = embed "../shared/respond.js"), (name = "shared-bounded-body", esModule = embed "../shared/bounded-body.js"), (name = "shared-request-scope", esModule = embed "../shared/request-scope.js"), (name = "shared-env", esModule = embed "../shared/env.js"), @@ -136,7 +141,7 @@ const loaderWorker :Workerd.Worker = ( const doOwnerNetworkWorker :Workerd.Worker = ( modules = [ (name = "worker", esModule = embed "do-owner-network.js"), - (name = "runtime-owner-endpoint", esModule = embed "_wdl-owner-endpoint.js"), + (name = "shared-owner-endpoint", esModule = embed "../shared/owner-endpoint.js"), (name = "shared-internal-auth", esModule = embed "../shared/internal-auth.js"), ], compatibilityDate = "2026-04-24", @@ -150,9 +155,11 @@ const tailWorker :Workerd.Worker = ( modules = [ (name = "worker", esModule = embed "tail-worker.js"), (name = "hex.js", esModule = embed "../shared/hex.js"), + (name = "errors.js", esModule = embed "../shared/errors.js"), (name = "shared-observability", esModule = embed "../shared/observability.js"), (name = "runtime-tail-forwarder", esModule = embed "tail-forwarder.js"), (name = "runtime-bindings-proxy", esModule = embed "bindings/proxy.js"), + (name = "shared-errors", esModule = embed "../shared/errors.js"), (name = "shared-respond", esModule = embed "../shared/respond.js"), (name = "shared-internal-auth", esModule = embed "../shared/internal-auth.js"), ], diff --git a/runtime/d1-client.js b/runtime/d1-client.js index 4081910..c907376 100644 --- a/runtime/d1-client.js +++ b/runtime/d1-client.js @@ -9,6 +9,10 @@ import { requestIdFromOptions } from "./_wdl-request-id.js"; const D1_SESSION_CONSTRAINT_FIRST_PRIMARY = "first-primary"; const D1_SESSION_CONSTRAINT_FIRST_UNCONSTRAINED = "first-unconstrained"; +const intrinsicReflectApply = Reflect.apply; +const intrinsicWeakMapGet = WeakMap.prototype.get; +const intrinsicWeakMapHas = WeakMap.prototype.has; +const intrinsicWeakMapSet = WeakMap.prototype.set; /** * @typedef {string | number | null | undefined | number[]} D1Param @@ -92,10 +96,25 @@ const databaseState = /** @type {WeakMap} */ (new WeakM const sessionState = /** @type {WeakMap} */ (new WeakMap()); const statementState = /** @type {WeakMap} */ (new WeakMap()); +/** @param {WeakMap} map @param {object} key */ +function weakMapGet(map, key) { + return intrinsicReflectApply(intrinsicWeakMapGet, map, [key]); +} + +/** @param {WeakMap} map @param {object} key */ +function weakMapHas(map, key) { + return intrinsicReflectApply(intrinsicWeakMapHas, map, [key]); +} + +/** @param {WeakMap} map @param {object} key @param {unknown} value */ +function weakMapSet(map, key, value) { + intrinsicReflectApply(intrinsicWeakMapSet, map, [key, value]); +} + /** @param {object} dbOrSession */ function rootDatabase(dbOrSession) { - if (databaseState.has(dbOrSession)) return dbOrSession; - const session = sessionState.get(dbOrSession); + if (weakMapHas(databaseState, dbOrSession)) return dbOrSession; + const session = /** @type {D1SessionState | undefined} */ (weakMapGet(sessionState, dbOrSession)); if (session) return session.db; throw new D1Error("D1_ERROR: invalid D1 database/session object", { code: "invalid-database", @@ -105,14 +124,14 @@ function rootDatabase(dbOrSession) { /** @param {object} db */ function requestIdFor(db) { - const state = databaseState.get(db); + const state = /** @type {D1DatabaseState | undefined} */ (weakMapGet(databaseState, db)); if (!state) return null; return requestIdFromOptions(state.requestIdOptions); } /** @param {object} statement */ function serializeStatement(statement) { - const state = statementState.get(statement); + const state = /** @type {D1StatementState | undefined} */ (weakMapGet(statementState, statement)); if (!state) { throw new D1Error("D1_ERROR: batch() expects D1PreparedStatement values", { code: "invalid-batch", @@ -143,7 +162,7 @@ function prepareFor(dbOrSession, statement) { */ async function sendFor(dbOrSession, mode, statements) { const db = rootDatabase(dbOrSession); - const state = databaseState.get(db); + const state = /** @type {D1DatabaseState | undefined} */ (weakMapGet(databaseState, db)); if (!state) throw new D1Error("D1_ERROR: invalid D1 database object", { code: "invalid-database", category: "invalid-request", @@ -181,7 +200,7 @@ async function batchFor(dbOrSession, statements) { /** @type {SerializedStatement[]} */ const serialized = []; for (const statement of statements) { - const state = statementState.get(statement); + const state = /** @type {D1StatementState | undefined} */ (weakMapGet(statementState, statement)); if (!state) { throw new D1Error("D1_ERROR: batch() expects D1PreparedStatement values", { code: "invalid-batch", @@ -208,13 +227,13 @@ export class D1PreparedStatement { * @param {object} [owner] */ constructor(dbOrSession, statement, params = [], owner = dbOrSession) { - statementState.set(this, { dbOrSession, statement, params, owner }); + weakMapSet(statementState, this, { dbOrSession, statement, params, owner }); } /** @param {...unknown} values */ bind(...values) { try { - const state = statementState.get(this); + const state = /** @type {D1StatementState | undefined} */ (weakMapGet(statementState, this)); if (!state) throw new Error("invalid prepared statement"); return new D1PreparedStatement( state.dbOrSession, @@ -232,7 +251,7 @@ export class D1PreparedStatement { /** @param {string} [columnName] */ async first(columnName) { - const state = statementState.get(this); + const state = /** @type {D1StatementState | undefined} */ (weakMapGet(statementState, this)); if (!state) throw new D1Error("D1_ERROR: invalid prepared statement", { code: "invalid-statement", category: "invalid-request", @@ -255,7 +274,7 @@ export class D1PreparedStatement { } async run() { - const state = statementState.get(this); + const state = /** @type {D1StatementState | undefined} */ (weakMapGet(statementState, this)); if (!state) throw new D1Error("D1_ERROR: invalid prepared statement", { code: "invalid-statement", category: "invalid-request", @@ -264,7 +283,7 @@ export class D1PreparedStatement { } async all() { - const state = statementState.get(this); + const state = /** @type {D1StatementState | undefined} */ (weakMapGet(statementState, this)); if (!state) throw new D1Error("D1_ERROR: invalid prepared statement", { code: "invalid-statement", category: "invalid-request", @@ -274,7 +293,7 @@ export class D1PreparedStatement { /** @param {{ columnNames?: boolean }} [options] */ async raw(options = {}) { - const state = statementState.get(this); + const state = /** @type {D1StatementState | undefined} */ (weakMapGet(statementState, this)); if (!state) throw new D1Error("D1_ERROR: invalid prepared statement", { code: "invalid-statement", category: "invalid-request", @@ -297,7 +316,7 @@ export class D1DatabaseSession { */ constructor(db, constraintOrBookmark = D1_SESSION_CONSTRAINT_FIRST_UNCONSTRAINED) { const normalized = String(constraintOrBookmark ?? "").trim() || D1_SESSION_CONSTRAINT_FIRST_UNCONSTRAINED; - sessionState.set(this, { db, bookmarkOrConstraint: normalized }); + weakMapSet(sessionState, this, { db, bookmarkOrConstraint: normalized }); } /** @param {unknown} statement */ @@ -311,7 +330,7 @@ export class D1DatabaseSession { } getBookmark() { - const state = sessionState.get(this); + const state = /** @type {D1SessionState | undefined} */ (weakMapGet(sessionState, this)); if (!state) return null; const { bookmarkOrConstraint } = state; if ( @@ -330,7 +349,7 @@ export class D1Database { * @param {{ requestId?: unknown, requestIdProvider?: unknown }} [options] */ constructor(stub, options = {}) { - databaseState.set(this, { + weakMapSet(databaseState, this, { stub, requestIdOptions: options, }); diff --git a/runtime/dispatch.js b/runtime/dispatch.js index a6313c0..97217b4 100644 --- a/runtime/dispatch.js +++ b/runtime/dispatch.js @@ -14,7 +14,11 @@ import { normalizeQueuedDispatchBody, normalizeScheduledDispatchBody, } from "runtime-lib"; -import { emitRuntimeTailEvent, fetchTailFields } from "runtime-tail-forwarder"; +import { + emitRuntimeTailEvent, + fetchTailFields, + startTailEnvelope, +} from "runtime-tail-forwarder"; import { _stringifyWorkflowBackendBodyForTest as stringifyWorkflowBackendBodyForTest, _stringifyWorkflowJsonForTest as stringifyWorkflowJsonForTest, @@ -135,35 +139,6 @@ function workflowTailFields(run) { }; } -/** @param {{ env: RuntimeEnv, ctx: RuntimeCtx, identity: RuntimeIdentity, event: string, fields: Record }} opts */ -function startTailEnvelope({ env, ctx, identity, event, fields }) { - const startedAt = Date.now(); - const startTailEvent = emitRuntimeTailEvent({ - env, ctx, identity, - event, - phase: "start", - fields, - }); - return { - /** @param {Record} extraFields */ - finish(extraFields) { - const durationMs = Date.now() - startedAt; - emitRuntimeTailEvent({ - env, ctx, identity, - event, - phase: "finish", - after: startTailEvent, - fields: { - ...fields, - ...extraFields, - duration_ms: durationMs, - }, - }); - return durationMs; - }, - }; -} - /** @param {WorkerDispatchArgs} args */ export async function handleFetchDispatch({ request, stub, scope, env, ctx, identity }) { const tail = startTailEnvelope({ @@ -180,10 +155,7 @@ export async function handleFetchDispatch({ request, stub, scope, env, ctx, iden return scope.respond(response); } catch (err) { scope.markError(err); - tail.finish({ - outcome: "error", - error: errorMessage(err), - }); + tail.finishError(err); return scope.respond(internalErrorResponse(502, "runtime_error", "Runtime error", scope.requestId)); } } diff --git a/runtime/dispatch/workflow-json.js b/runtime/dispatch/workflow-json.js index 4666959..03ea2e6 100644 --- a/runtime/dispatch/workflow-json.js +++ b/runtime/dispatch/workflow-json.js @@ -1,5 +1,6 @@ -const WORKFLOW_RESULT_BYTES_MAX = 1024 * 1024; -const WORKFLOW_BACKEND_REQUEST_BYTES_MAX = 2 * 1024 * 1024; +export const WORKFLOW_RESULT_BYTES_MAX = 1024 * 1024; +export const WORKFLOW_BACKEND_REQUEST_BYTES_MAX = 2 * 1024 * 1024; +export const WORKFLOW_PAYLOAD_TOO_LARGE_CODE = "workflow_payload_too_large"; const WORKFLOW_JSON_ENCODE_CHARS = 8192; const WORKFLOW_JSON_FLUSH_CHARS = 8192; const utf8Encoder = new TextEncoder(); @@ -20,7 +21,7 @@ export function workflowStepError(code, message) { */ function workflowPayloadTooLarge(kind, maxBytes = WORKFLOW_RESULT_BYTES_MAX) { return workflowStepError( - "workflow_payload_too_large", + WORKFLOW_PAYLOAD_TOO_LARGE_CODE, `Workflow ${kind} exceeds the ${maxBytes} byte limit` ); } @@ -255,7 +256,6 @@ export function stringifyWorkflowResult(value, kind) { return stringifyWorkflowJson(value, kind, WORKFLOW_RESULT_BYTES_MAX); } -/** @lintignore data-URL unit tests import this hook from a rewritten module. */ /** * @param {unknown} value * @param {number} [maxBytes] @@ -264,7 +264,6 @@ export function _stringifyWorkflowJsonForTest(value, maxBytes = WORKFLOW_RESULT_ return stringifyWorkflowJson(value, "test value", maxBytes); } -/** @lintignore data-URL unit tests import this hook from a rewritten module. */ /** @param {string} path @param {unknown} body */ export function _stringifyWorkflowBackendBodyForTest(path, body) { return workflowBackendBody(path, body); diff --git a/runtime/do-client.js b/runtime/do-client.js index 48e8451..8ef3198 100644 --- a/runtime/do-client.js +++ b/runtime/do-client.js @@ -3,25 +3,41 @@ import { DO_INVOKE_URL, connectHeaders, doOwnerHintCacheKey, - dispatchDoRequestWithOwnerHint, + dispatchDoConnectWithHintCache, + dispatchDoInvokeWithHintCache, fetchInvokeInit, isWebSocketUpgrade, - retryableOwnerRaceResponse, + replayOwnerUnavailableForFetch, rpcInvokeInit, rpcResultFromResponse, - staleDoOwnerHintResponse, - stripOwnerHintHeaders, - withoutOwnerHintOptIn, } from "./_wdl-do-transport.js"; import { createOwnerHintCache } from "./_wdl-owner-hint-cache.js"; import { requestIdFromOptions } from "./_wdl-request-id.js"; const ownerHintCache = createOwnerHintCache(); +const intrinsicObjectHasOwn = Object.hasOwn; +const intrinsicReflectApply = Reflect.apply; +const intrinsicStringIsWellFormed = String.prototype.isWellFormed; -/** @param {Request} request */ -function replayOwnerUnavailableForFetch(request) { - const method = request.method.toUpperCase(); - return method === "GET" || method === "HEAD"; +/** @param {object} object @param {PropertyKey} key */ +function objectHasOwn(object, key) { + return intrinsicReflectApply(intrinsicObjectHasOwn, undefined, [object, key]); +} + +/** @param {string} value */ +function isWellFormedUnicodeString(value) { + return intrinsicReflectApply(intrinsicStringIsWellFormed, value, []); +} + +/** @param {unknown} value @param {string} method */ +function requireObjectIdString(value, method) { + if (typeof value !== "string" || !value) { + throw new TypeError(`DurableObjectNamespace.${method}() requires a non-empty string`); + } + if (!isWellFormedUnicodeString(value)) { + throw new TypeError(`DurableObjectNamespace.${method}() requires well-formed Unicode`); + } + return value; } /** @@ -84,7 +100,7 @@ class DurableObjectStub { const real = Reflect.get(target, prop, receiver); if (real !== undefined) return real; /** @param {...unknown} args */ - const method = (...args) => target.namespace.rpcObject(target.id.name, prop, args, target.namespace.requestId()); + const method = (...args) => target.namespace.rpcObject(target.id.name, prop, args); return method; }, }); @@ -93,7 +109,7 @@ class DurableObjectStub { /** @param {RequestInfo | URL} input @param {RequestInit} [init] */ async fetch(input, init = undefined) { const request = new Request(input, init); - return await this.namespace.fetchObject(this.id.name, request, this.namespace.requestId()); + return await this.namespace.fetchObject(this.id.name, request); } } @@ -147,12 +163,12 @@ export class DurableObjectNamespace { */ constructor(binding, options = {}) { const isMetadataBinding = binding && typeof binding === "object" && ( - Object.hasOwn(binding, "ns") || - Object.hasOwn(binding, "worker") || - Object.hasOwn(binding, "version") || - Object.hasOwn(binding, "doStorageId") || - Object.hasOwn(binding, "binding") || - Object.hasOwn(binding, "className") + objectHasOwn(binding, "ns") || + objectHasOwn(binding, "worker") || + objectHasOwn(binding, "version") || + objectHasOwn(binding, "doStorageId") || + objectHasOwn(binding, "binding") || + objectHasOwn(binding, "className") ); if (binding && !isMetadataBinding) { this.#setBindingProxy(/** @type {DurableObjectBindingProxy} */ (binding)); @@ -179,12 +195,13 @@ export class DurableObjectNamespace { } } - requestId() { - return requestIdFromOptions(this.#requestIdOptions, { providerFirst: false }); + #currentRequestId() { + return requestIdFromOptions(this.#requestIdOptions); } - /** @param {string} objectName @param {Request} request @param {string | null} [requestId] */ - async fetchObject(objectName, request, requestId = null) { + /** @param {string} objectName @param {Request} request */ + async fetchObject(objectName, request) { + const requestId = this.#currentRequestId(); if (this.#bindingFetchObject && !isWebSocketUpgrade(request)) { return await this.#bindingFetchObject(objectName, request, requestId); } @@ -201,32 +218,29 @@ export class DurableObjectNamespace { method: "GET", headers: connectHeaders(props, objectName, request, requestId), }; - const response = await this.#fetchWithOwnerHint( + return await this.#dispatchWithOwnerHint( + dispatchDoConnectWithHintCache, DO_CONNECT_URL, "/internal/do/connect", init, doOwnerHintCacheKey(props, objectName), false ); - return stripOwnerHintHeaders(response); } const init = await fetchInvokeInit(props, objectName, request, requestId); - const response = await this.#fetchWithOwnerHint( + return await this.#dispatchWithOwnerHint( + dispatchDoInvokeWithHintCache, DO_INVOKE_URL, "/internal/do/invoke", init, doOwnerHintCacheKey(props, objectName), replayOwnerUnavailableForFetch(request) ); - if (await retryableOwnerRaceResponse(response)) { - ownerHintCache.delete(doOwnerHintCacheKey(props, objectName)); - return stripOwnerHintHeaders(await backend.fetch(DO_INVOKE_URL, withoutOwnerHintOptIn(init))); - } - return stripOwnerHintHeaders(response); } - /** @param {string} objectName @param {string} method @param {unknown[]} args @param {string | null} [requestId] */ - async rpcObject(objectName, method, args, requestId = null) { + /** @param {string} objectName @param {string} method @param {unknown[]} args */ + async rpcObject(objectName, method, args) { + const requestId = this.#currentRequestId(); if (this.#bindingRpcObject) { return await this.#bindingRpcObject(objectName, method, args, requestId); } @@ -236,32 +250,26 @@ export class DurableObjectNamespace { } const props = requireBindingProps(this.#props); const init = rpcInvokeInit(props, objectName, method, args, requestId); - const response = await this.#fetchWithOwnerHint( + const response = await this.#dispatchWithOwnerHint( + dispatchDoInvokeWithHintCache, DO_INVOKE_URL, "/internal/do/invoke", init, doOwnerHintCacheKey(props, objectName), false ); - let routed = response; - if (await retryableOwnerRaceResponse(response)) { - // Owner-hint following already happened above. This retry is only for - // generation races after reaching an owner, so it falls back through the - // router with hint opt-in disabled. - ownerHintCache.delete(doOwnerHintCacheKey(props, objectName)); - routed = await backend.fetch(DO_INVOKE_URL, withoutOwnerHintOptIn(init)); - } - return await rpcResultFromResponse(stripOwnerHintHeaders(routed)); + return await rpcResultFromResponse(response); } /** + * @param {typeof dispatchDoInvokeWithHintCache | typeof dispatchDoConnectWithHintCache} dispatch * @param {string} routerUrl * @param {string} ownerPath * @param {RequestInit} init * @param {string} hintKey * @param {boolean} replayOwnerUnavailable */ - async #fetchWithOwnerHint(routerUrl, ownerPath, init, hintKey, replayOwnerUnavailable) { + async #dispatchWithOwnerHint(dispatch, routerUrl, ownerPath, init, hintKey, replayOwnerUnavailable) { const backend = this.#backend; const ownerNetwork = this.#ownerNetwork; if (!backend || typeof backend.fetch !== "function") { @@ -273,36 +281,26 @@ export class DurableObjectNamespace { : null; /** @type {import("./_wdl-do-transport.js").DoFetch} */ const routerFetch = (url, requestInit) => backend.fetch(url, requestInit); - return await dispatchDoRequestWithOwnerHint({ + return await dispatch({ routerFetch, routerUrl, ownerFetch, ownerPath, init, - cachedHint: /** @type {import("./_wdl-do-transport.js").DoOwnerHint | null} */ ( - ownerHintCache.get(hintKey) - ), - rememberHint: (hint) => ownerHintCache.set(hintKey, hint), - clearHint: () => ownerHintCache.delete(hintKey), - staleCachedResponse: staleDoOwnerHintResponse, + cache: ownerHintCache, + hintKey, replayOwnerUnavailable, }); } /** @param {string} name */ idFromName(name) { - if (typeof name !== "string" || !name) { - throw new TypeError("DurableObjectNamespace.idFromName() requires a non-empty string"); - } - return new DurableObjectId(name); + return new DurableObjectId(requireObjectIdString(name, "idFromName")); } /** @param {string} value */ idFromString(value) { - if (typeof value !== "string" || !value) { - throw new TypeError("DurableObjectNamespace.idFromString() requires a non-empty string"); - } - return new DurableObjectId(value); + return new DurableObjectId(requireObjectIdString(value, "idFromString")); } newUniqueId() { diff --git a/runtime/do-owner-network.js b/runtime/do-owner-network.js index 3eb9b14..4151274 100644 --- a/runtime/do-owner-network.js +++ b/runtime/do-owner-network.js @@ -1,4 +1,4 @@ -import { validOwnerEndpointForService } from "runtime-owner-endpoint"; +import { validOwnerEndpointForService } from "shared-owner-endpoint"; import { withInternalAuth } from "shared-internal-auth"; /** diff --git a/runtime/index.js b/runtime/index.js index d5e79e2..20fc17c 100644 --- a/runtime/index.js +++ b/runtime/index.js @@ -11,11 +11,9 @@ import { parseDispatchWorkerId } from "shared-worker-id"; import { handleFetchDispatch, } from "runtime-dispatch"; -import { createLoaderCallback } from "runtime-load"; +import { getLoadedWorkerStub } from "runtime-load"; import { bindRuntime, - evictSiblings, - recordLoadedWorker, runtimeServiceAllowsNamespace, } from "runtime-state"; @@ -64,23 +62,13 @@ export default class Runtime extends WorkerEntrypoint { return scope.respond(jsonError(403, "runtime_pool_mismatch", "Worker namespace is not allowed in this runtime pool")); } - const baseCallback = createLoaderCallback({ + // Gateway-routed traffic is the authoritative "this version is active" + // signal. Service-binding cold-loads use the same helper without eviction. + const { stub } = getLoadedWorkerStub({ requestId: scope.requestId, env, ctx, ns: namespace, worker: workerName, version, workerId, metrics: runtime.metrics, log: runtime.log, - }); - // Factory fires only on cache miss. Gateway-routed traffic is the - // authoritative "this version is active" signal, so this is the - // one place we evict historical siblings; service-binding cold - // loads record but do not evict. - const stub = env.LOADER.get(workerId, async () => { - const code = await baseCallback(); - recordLoadedWorker(workerId); - ctx.waitUntil( - evictSiblings({ env, workerId, log: runtime.log }) - .catch(() => {}) - ); - return code; + evictOnLoad: true, }); const forwardRequest = new Request(request); diff --git a/runtime/internal.js b/runtime/internal.js index 6d90a57..51c0fe5 100644 --- a/runtime/internal.js +++ b/runtime/internal.js @@ -2,7 +2,7 @@ // runtime/index.js on :8081; scheduler/workflows call this worker on :8088. import { WorkerEntrypoint } from "cloudflare:workers"; -import { formatWorkerId, parseDispatchWorkerId } from "shared-worker-id"; +import { parseDispatchWorkerId } from "shared-worker-id"; import { internalErrorResponse, jsonError, @@ -21,11 +21,9 @@ import { readWorkflowNotifyDispatch, readWorkflowRunDispatch, } from "runtime-dispatch"; -import { createLoaderCallback } from "runtime-load"; +import { getLoadedWorkerStub } from "runtime-load"; import { bindRuntime, - evictSiblings, - recordLoadedWorker, runtimeServiceAllowsNamespace, } from "runtime-state"; @@ -51,24 +49,12 @@ export { InternalAuthBackend } from "runtime-bindings-internal-auth-backend"; * @param {{ env: RuntimeInternalEnv, ctx: { waitUntil(promise: Promise): void }, runtime: RuntimeBinding, scope: RuntimeScope, namespace: string, workerName: string, version: string, evictOnLoad?: boolean }} opts */ function loadStub({ env, ctx, runtime, scope, namespace, workerName, version, evictOnLoad = false }) { - const workerId = formatWorkerId({ namespace, worker: workerName, version }); - const baseCallback = createLoaderCallback({ + return getLoadedWorkerStub({ requestId: scope.requestId, env, ctx, - ns: namespace, worker: workerName, version, workerId, + ns: namespace, worker: workerName, version, metrics: runtime.metrics, log: runtime.log, + evictOnLoad, }); - const stub = env.LOADER.get(workerId, async () => { - const code = await baseCallback(); - recordLoadedWorker(workerId); - if (evictOnLoad) { - ctx.waitUntil( - evictSiblings({ env, workerId, log: runtime.log }) - .catch(() => {}) - ); - } - return code; - }); - return { workerId, stub }; } /** @param {{ serviceName: string }} runtime @param {string} namespace */ diff --git a/runtime/lib.js b/runtime/lib.js index 0d6b977..56f290c 100644 --- a/runtime/lib.js +++ b/runtime/lib.js @@ -1,11 +1,22 @@ // Pure helpers for the runtime worker. -import { isWorkerdExperimentalCompatFlag } from "shared-workerd-compat-flags"; +import { base64ToBytes, bytesToBase64 } from "shared-base64"; +import { WORKER_NAME_RE, isValidRouteNs } from "shared-ns-pattern"; +import { parseVersion } from "shared-version"; +import { + DEFAULT_DYNAMIC_WORKER_COMPATIBILITY_DATE, + ENHANCED_ERROR_SERIALIZATION_DEFAULT_DATE, + ENHANCED_ERROR_SERIALIZATION_FLAG, + LEGACY_ERROR_SERIALIZATION_FLAG, + MIN_DYNAMIC_WORKER_COMPATIBILITY_DATE, + isWorkerdExperimentalCompatFlag, +} from "shared-workerd-compat-flags"; -const BASE64_CHUNK_SIZE = 0x8000; const utf8Encoder = new TextEncoder(); const utf8Decoder = new TextDecoder(); +export { base64ToBytes, bytesToBase64 }; + // Coerce a KV.put value into Uint8Array. ReadableStream is handled by the // caller (await body); here we stick to sync inputs. /** @param {unknown} value */ @@ -19,23 +30,6 @@ export function toBytes(value) { throw new Error("KV put: value must be string | ArrayBuffer | typed array | ReadableStream"); } -/** @param {Uint8Array} bytes @returns {string} */ -export function bytesToBase64(bytes) { - let binary = ""; - for (let offset = 0; offset < bytes.length; offset += BASE64_CHUNK_SIZE) { - binary += String.fromCharCode(...bytes.subarray(offset, offset + BASE64_CHUNK_SIZE)); - } - return btoa(binary); -} - -/** @param {string} value @returns {Uint8Array} */ -export function base64ToBytes(value) { - const binary = atob(value); - const out = new Uint8Array(binary.length); - for (let i = 0; i < binary.length; i += 1) out[i] = binary.charCodeAt(i); - return out; -} - export const MAX_QUEUE_DELAY_SECONDS = 86_400; export const QUEUE_CONTENT_TYPES = Object.freeze({ JSON: "json", @@ -110,7 +104,7 @@ export function workerQueueAttempts(internalAttempts) { return Number.isFinite(n) && n >= 0 ? Math.floor(n) + 1 : 1; } -// Mirror of control/lib.js#deepFreeze — bundle meta is immutable on the +// Mirror of control/bundle.js#deepFreeze — bundle meta is immutable on the // wire, so the post-parse object should be too. /** @template T @param {T} obj @returns {T} */ function deepFreeze(obj) { @@ -120,14 +114,6 @@ function deepFreeze(obj) { return Object.freeze(obj); } -const DEFAULT_COMPATIBILITY_DATE = "2026-04-24"; -const ENHANCED_ERROR_SERIALIZATION_DEFAULT_DATE = "2026-04-21"; -const ENHANCED_ERROR_SERIALIZATION_FLAG = "enhanced_error_serialization"; -const ROUTE_NS_RE = /^(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?|__system__)$/; -const RESERVED_TENANT_NS = new Set(["admin"]); -const WORKER_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,254}$/; -const VERSION_RE = /^v[1-9][0-9]*$/; - // Loaded workers may declare an older compatibilityDate than the platform // workers. Keep enhanced error serialization as a floor only before the date // where workerd made it the default; newer dates reject the explicit flag. @@ -166,6 +152,19 @@ function mergeCompatFlags(userFlags, compatibilityDate) { `meta.compatibilityFlags contains experimental workerd flag ${JSON.stringify(f)}, which WDL does not support for tenant workers` ); } + if (f === LEGACY_ERROR_SERIALIZATION_FLAG) { + throw new Error( + `meta.compatibilityFlags contains unsupported flag ${JSON.stringify(f)}; WDL requires enhanced error serialization` + ); + } + if ( + f === ENHANCED_ERROR_SERIALIZATION_FLAG && + compatibilityDate >= ENHANCED_ERROR_SERIALIZATION_DEFAULT_DATE + ) { + throw new Error( + `meta.compatibilityFlags contains ${JSON.stringify(f)}, which became the workerd default on ${ENHANCED_ERROR_SERIALIZATION_DEFAULT_DATE} and must not be specified for compatibilityDate ${compatibilityDate}` + ); + } out.push(f); } for (const f of platformFloorCompatFlags(compatibilityDate)) { @@ -240,7 +239,12 @@ export function bundleToWorkerCode(hash) { // methods cannot shadow RPC method names on bindings. const compatibilityDate = typeof meta.compatibilityDate === "string" && meta.compatibilityDate ? meta.compatibilityDate - : DEFAULT_COMPATIBILITY_DATE; + : DEFAULT_DYNAMIC_WORKER_COMPATIBILITY_DATE; + if (compatibilityDate < MIN_DYNAMIC_WORKER_COMPATIBILITY_DATE) { + throw new Error( + `meta.compatibilityDate ${compatibilityDate} is older than WDL supports (${MIN_DYNAMIC_WORKER_COMPATIBILITY_DATE})` + ); + } return { compatibilityDate, @@ -313,8 +317,8 @@ function requiredPatternString(body, field, pattern, description) { /** @param {Record | null} body */ function requiredRouteNs(body) { - const ns = requiredPatternString(body, "ns", ROUTE_NS_RE, "a route namespace"); - if (RESERVED_TENANT_NS.has(ns)) { + const ns = requiredString(body, "ns"); + if (!isValidRouteNs(ns)) { throw new Error("ns must be a route namespace"); } return ns; @@ -327,7 +331,11 @@ function requiredWorkerName(body) { /** @param {Record | null} body */ function requiredVersion(body) { - return requiredPatternString(body, "frozenVersion", VERSION_RE, "an immutable worker version"); + const version = requiredString(body, "frozenVersion"); + if (parseVersion(version) == null) { + throw new Error("frozenVersion must be an immutable worker version"); + } + return version; } /** @param {Record | null} body @param {string} field */ diff --git a/runtime/load.js b/runtime/load.js index d33574e..88904b0 100644 --- a/runtime/load.js +++ b/runtime/load.js @@ -5,7 +5,8 @@ import { bundleToWorkerCode } from "runtime-lib"; import { formatError } from "shared-observability"; import { withInternalAuth } from "shared-internal-auth"; import { discardResponseBody } from "shared-respond"; -import { parseRuntimeLoadWorkerId } from "shared-worker-id"; +import { formatWorkerId, parseRuntimeLoadWorkerId } from "shared-worker-id"; +import { evictSiblings, recordLoadedWorker } from "runtime-state"; import { analyzeRuntimeMeta, injectRuntimeModulesForHostBindings, @@ -348,3 +349,56 @@ export function createLoaderCallback({ requestId, env, ctx, ns, worker, version, return workerCode; }; } + +/** + * @template TStub + * @param {{ + * requestId: string | null, + * env: RuntimeLoaderEnv & { LOADER: { get(id: string, factory: () => Promise): TStub } }, + * ctx: { waitUntil(promise: Promise): void }, + * ns: string, + * worker: string, + * version: string, + * workerId?: string, + * metrics?: RuntimeLoaderMetrics | null, + * log?: ((level: string, event: string, fields?: Record) => void) | null, + * evictOnLoad?: boolean, + * }} options + * @returns {{ workerId: string, stub: TStub }} + */ +export function getLoadedWorkerStub({ + requestId, + env, + ctx, + ns, + worker, + version, + workerId = formatWorkerId({ namespace: ns, worker, version }), + metrics = null, + log = null, + evictOnLoad = false, +}) { + const baseCallback = createLoaderCallback({ + requestId, + env, + ctx, + ns, + worker, + version, + workerId, + metrics, + log, + }); + const stub = env.LOADER.get(workerId, async () => { + const code = await baseCallback(); + recordLoadedWorker(workerId); + if (evictOnLoad) { + ctx.waitUntil( + evictSiblings({ env, workerId, log }) + .catch(() => {}) + ); + } + return code; + }); + return { workerId, stub }; +} diff --git a/runtime/load/code-budget.js b/runtime/load/code-budget.js index 120714a..b92f2b1 100644 --- a/runtime/load/code-budget.js +++ b/runtime/load/code-budget.js @@ -10,6 +10,8 @@ import { rewriteCloudflareWorkflowsImports, } from "runtime-load-module-rewrite"; import { + HOST_BINDING_RUNTIME_MODULE_NAME, + HOST_BINDING_RUNTIME_SOURCE, generateAbortShimWrapperModule, generateHostBindingWrapperModule, } from "runtime-load-wrapper-generate"; @@ -19,6 +21,9 @@ import { export const WORKER_LOADER_CODE_MAX_BYTES = 64 * 1024 * 1024; const D1_DATA_FIELD_MODULE_NAME = "_wdl-d1-data-field.js"; +const NODEJS_ALS_COMPATIBILITY_FLAG = "nodejs_als"; +const NODEJS_COMPATIBILITY_FLAG = "nodejs_compat"; +const NO_NODEJS_ALS_COMPATIBILITY_FLAG = "no_nodejs_als"; const utf8Decoder = new TextDecoder(); /** @@ -281,6 +286,9 @@ function runtimeInjectedModuleSources(mainModule, meta, runtimeSources, plan = a addModules(injections.workflowsModuleInjections); } out.set(WORKFLOWS_MODULE_NAME, WORKFLOWS_MODULE_SOURCE); + if (plan.needsHostBindingWrapper) { + out.set(HOST_BINDING_RUNTIME_MODULE_NAME, HOST_BINDING_RUNTIME_SOURCE); + } // `_wdl-wrapper.js` is always injected: host bindings use the larger wrapper, // and otherwise the abort shim still rewrites the user main module. out.set( @@ -327,6 +335,18 @@ export function injectRuntimeModulesForHostBindings( for (const [name, source] of runtimeInjectedModuleSources(originalMain, meta, runtimeSources, plan)) { workerCode.modules[name] = source; } + if (plan.needsHostBindingWrapper) { + const flags = Array.isArray(workerCode.compatibilityFlags) + ? /** @type {string[]} */ (workerCode.compatibilityFlags) + : []; + if (!flags.includes(NODEJS_COMPATIBILITY_FLAG)) { + const normalized = flags.filter((flag) => flag !== NO_NODEJS_ALS_COMPATIBILITY_FLAG); + if (!normalized.includes(NODEJS_ALS_COMPATIBILITY_FLAG)) { + normalized.push(NODEJS_ALS_COMPATIBILITY_FLAG); + } + workerCode.compatibilityFlags = normalized; + } + } workerCode.mainModule = "_wdl-wrapper.js"; return workerCode; } diff --git a/runtime/load/env-build.js b/runtime/load/env-build.js index 161aea9..c478f08 100644 --- a/runtime/load/env-build.js +++ b/runtime/load/env-build.js @@ -1,11 +1,14 @@ import { BINDING_NAME_RE, + D1_DATABASE_ID_RE, RESERVED_OBJECT_KEYS, WDL_RESERVED_BINDING_RE, WDL_RESERVED_ENTRYPOINT_RE, isValidJsIdentifier, isValidJsClassDeclarationName, + isValidRuntimeLoadNs, } from "shared-ns-pattern"; +import { parseVersion } from "shared-version"; const DO_BACKEND_BINDING = "__WDL_DO_BACKEND__"; const DO_OWNER_NETWORK_BINDING = "__WDL_DO_OWNER_NETWORK__"; @@ -91,8 +94,11 @@ function materializeQueueBinding({ name, spec, ns, ctx }) { /** @param {RuntimeBindingMaterializerArgs} args */ function materializeD1Binding({ name, spec, ns, ctx }) { const databaseId = spec.databaseId; - if (typeof databaseId !== "string" || !databaseId) { - throw new Error(`Binding "${name}" is a D1 binding but missing databaseId`); + if (!isValidRuntimeLoadNs(ns)) { + throw new Error(`Binding "${name}" is a D1 binding but has invalid namespace`); + } + if (typeof databaseId !== "string" || !D1_DATABASE_ID_RE.test(databaseId)) { + throw new Error(`Binding "${name}" is a D1 binding but has invalid databaseId`); } return { value: ctx.exports.D1Database({ @@ -164,6 +170,9 @@ function materializeServiceBinding({ name, spec, ns, worker, nsSecrets, workerSe `Binding "${name}" is a service binding but missing service/version (control should have pinned these at deploy time)` ); } + if (parseVersion(spec.version) == null) { + throw new Error(`Binding "${name}" is a service binding but has invalid version (redeploy ${ns}/${worker})`); + } if (spec.ns != null && (typeof spec.ns !== "string" || !spec.ns)) { throw new Error(`Binding "${name}" is a service binding but has invalid ns (redeploy ${ns}/${worker})`); } @@ -178,11 +187,16 @@ function materializeServiceBinding({ name, spec, ns, worker, nsSecrets, workerSe // Secrets-only (not vars): a credential moved from secrets → vars // stops reaching the target instead of silently continuing. // Missing keys are dropped; control warned at deploy. + const requiredCallerSecrets = Array.isArray(spec.requiredCallerSecrets) + ? spec.requiredCallerSecrets + : []; /** @type {Record | undefined} */ - let callerSecrets; - if (Array.isArray(spec.requiredCallerSecrets) && spec.requiredCallerSecrets.length) { - callerSecrets = {}; - for (const k of spec.requiredCallerSecrets) { + const callerSecrets = requiredCallerSecrets.length + // workerd's JSRPC props serializer rejects null-prototype objects. + ? {} + : undefined; + if (callerSecrets) { + for (const k of requiredCallerSecrets) { if (typeof k !== "string") continue; if (Object.hasOwn(workerSecrets, k)) { callerSecrets[k] = workerSecrets[k]; diff --git a/runtime/load/module-rewrite.js b/runtime/load/module-rewrite.js index ca85636..7097513 100644 --- a/runtime/load/module-rewrite.js +++ b/runtime/load/module-rewrite.js @@ -26,6 +26,7 @@ export const HOST_BINDING_RESERVED_MODULE_NAMES = Object.freeze([ "_wdl-owner-hint-cache.js", "_wdl-request-id.js", "_wdl-workflows-client.js", + "_wdl-host-wrapper-runtime.js", "_wdl-wrapper.js", ]); export const HOST_BINDING_RESERVED_MODULES = new Set(HOST_BINDING_RESERVED_MODULE_NAMES); diff --git a/runtime/load/wrapper-generate.js b/runtime/load/wrapper-generate.js index 1235257..0f4f2e3 100644 --- a/runtime/load/wrapper-generate.js +++ b/runtime/load/wrapper-generate.js @@ -1,3 +1,113 @@ +const WDL_ABORT_SHIM_SOURCE = `// Reserved name (WDL_RESERVED_ENTRYPOINT_RE) — control rejects user +// [[exports]] / [[services]] matches; implicit user exports collide +// silently via the export-* shadow rule. +export class __WdlAbort__ extends WorkerEntrypoint { + abort(reason) { + abortIsolate(reason ?? "wdl-evict"); + } +}`; + +const DEFAULT_EXPORT_SOURCE_SNIPPET = "const source = Function.prototype.toString.call(raw);"; +const DEFAULT_EXPORT_CLASS_TEST_SOURCE = "/^\\s*class\\b/.test(source)"; + +export const HOST_BINDING_RUNTIME_MODULE_NAME = "_wdl-host-wrapper-runtime.js"; +export const HOST_BINDING_RUNTIME_SOURCE = ` +import { AsyncLocalStorage } from "node:async_hooks"; +import { sanitizeRequestId } from "./_wdl-request-id.js"; + +const IntrinsicObject = Object; +const IntrinsicProxy = Proxy; +const IntrinsicReflect = Reflect; +const IntrinsicSymbol = Symbol; +const intrinsicAsyncLocalStorageGetStore = AsyncLocalStorage.prototype.getStore; +const intrinsicAsyncLocalStorageRun = AsyncLocalStorage.prototype.run; +const intrinsicArrayForEach = Array.prototype.forEach; +const intrinsicFunctionToString = Function.prototype.toString; +const intrinsicHeadersGet = Headers.prototype.get; +const intrinsicObjectDefineProperty = Object.defineProperty; +const intrinsicObjectEntries = Object.entries; +const intrinsicObjectKeys = Object.keys; +const intrinsicReflectApply = Reflect.apply; +const intrinsicReflectGet = Reflect.get; +const intrinsicRegExpTest = RegExp.prototype.test; +const intrinsicRequestHeadersGetter = Object.getOwnPropertyDescriptor(Request.prototype, "headers").get; + +export function applyFunction(fn, receiver, args) { + return intrinsicReflectApply(fn, receiver, args); +} + +export function createPrivateSymbol(description) { + return IntrinsicSymbol(description); +} + +export function createProxy(target, handler) { + return new IntrinsicProxy(target, handler); +} + +export function defineProperty(target, property, descriptor) { + return intrinsicReflectApply(intrinsicObjectDefineProperty, IntrinsicObject, [target, property, descriptor]); +} + +export function forEachArray(values, callback) { + intrinsicReflectApply(intrinsicArrayForEach, values, [callback]); +} + +export function forEachObjectEntry(record, callback) { + const entries = intrinsicReflectApply(intrinsicObjectEntries, IntrinsicObject, [record]); + forEachArray(entries, (entry) => callback(entry[0], entry[1])); +} + +export function functionSource(fn) { + return intrinsicReflectApply(intrinsicFunctionToString, fn, []); +} + +export function objectKeys(record) { + return intrinsicReflectApply(intrinsicObjectKeys, IntrinsicObject, [record]); +} + +export function reflectGet(target, property, receiver) { + return intrinsicReflectApply(intrinsicReflectGet, IntrinsicReflect, [target, property, receiver]); +} + +export function regexpTest(regexp, value) { + return intrinsicReflectApply(intrinsicRegExpTest, regexp, [value]); +} + +export function requestIdFromEventArg(arg) { + if (!arg || typeof arg !== "object") return null; + try { + const headers = intrinsicReflectApply(intrinsicRequestHeadersGetter, arg, []); + const requestId = intrinsicReflectApply(intrinsicHeadersGet, headers, ["x-request-id"]); + return sanitizeRequestId(requestId); + } catch { + return null; + } +} + +export function createRequestContext() { + const requestContext = new AsyncLocalStorage(); + return { + currentRequestId() { + const store = intrinsicReflectApply(intrinsicAsyncLocalStorageGetStore, requestContext, []); + const requestId = store?.requestId; + return typeof requestId === "string" && requestId ? requestId : null; + }, + runWithRequestContext(requestId, callback) { + const activeStore = intrinsicReflectApply(intrinsicAsyncLocalStorageGetStore, requestContext, []); + if (activeStore !== undefined) { + return intrinsicReflectApply(callback, undefined, []); + } + const canonicalRequestId = sanitizeRequestId(requestId); + return intrinsicReflectApply(intrinsicAsyncLocalStorageRun, requestContext, [ + { requestId: canonicalRequestId }, + callback, + ]); + }, + }; +} + +`; + /** @param {string} userMainSpecifier */ export function generateAbortShimWrapperModule(userMainSpecifier) { const userMain = JSON.stringify(`./${userMainSpecifier}`); @@ -6,14 +116,7 @@ import * as user from ${userMain}; import { WorkerEntrypoint, abortIsolate } from "cloudflare:workers"; export * from ${userMain}; -// Reserved name (WDL_RESERVED_ENTRYPOINT_RE) — control rejects user -// [[exports]] / [[services]] matches; implicit user exports collide -// silently via the export-* shadow rule. -export class __WdlAbort__ extends WorkerEntrypoint { - abort(reason) { - abortIsolate(reason ?? "wdl-evict"); - } -} +${WDL_ABORT_SHIM_SOURCE} export class __WdlWorkflowNotify__ extends WorkerEntrypoint { fetch() { @@ -25,8 +128,8 @@ const raw = user.default; let wrappedDefault = raw; if (typeof raw === "function") { - const source = Function.prototype.toString.call(raw); - if (!/^\\s*class\\b/.test(source)) { + ${DEFAULT_EXPORT_SOURCE_SNIPPET} + if (!${DEFAULT_EXPORT_CLASS_TEST_SOURCE}) { wrappedDefault = { fetch(request, env, ctx) { return raw.call(undefined, request, env, ctx); @@ -63,38 +166,34 @@ export function generateHostBindingWrapperModule(userMainSpecifier, d1Bindings, const starExport = hidesRawEnvExports ? "// Internal Fetcher bindings are present; only wrapped entrypoints are re-exported." : `export * from ${userMain};`; - const namedEntrypoints = entrypointNames.map((/** @type {string} */ name) => ` -export class ${name} extends user.${name} { - constructor(ctx, env) { - const requestContext = createRequestContext(); - super(ctx, wrapEnv(env, requestContext)); - return wrapClassInstance(this, requestContext); - } -} + const namedEntrypoints = entrypointNames.map((/** @type {string} */ name, index) => ` +const __WdlWrappedEntrypoint${index}__ = ({ + [${JSON.stringify(name)}]: class extends __WdlUserModule__.${name} { + constructor(ctx, env) { + super(ctx, wrapEnv(env)); + return wrapClassInstance(this); + } + }, +})[${JSON.stringify(name)}]; +export { __WdlWrappedEntrypoint${index}__ as ${name} }; `).join(""); return ` -import * as user from ${userMain}; import { WorkerEntrypoint, abortIsolate } from "cloudflare:workers"; +import * as __WdlHostRuntime__ from "./${HOST_BINDING_RUNTIME_MODULE_NAME}"; ${d1Import} ${r2Import} ${doImport} ${workflowImport} -// Local wrapper subclasses intentionally shadow same-name user exports, so -// declared entrypoints get facade-aware env even when star exports are present. +import * as __WdlUserModule__ from ${userMain}; +// Explicit aliases replace same-name star exports without declaring tenant +// entrypoint names in the generated module's local binding scope. ${starExport} -// Reserved name (WDL_RESERVED_ENTRYPOINT_RE) — control rejects user -// [[exports]] / [[services]] matches; implicit user exports collide -// silently via the export-* shadow rule. -export class __WdlAbort__ extends WorkerEntrypoint { - abort(reason) { - abortIsolate(reason ?? "wdl-evict"); - } -} +${WDL_ABORT_SHIM_SOURCE} export class __WdlWorkflowNotify__ extends WorkerEntrypoint { - async fetch(request) { - return await notifyWorkflowCallback(request, wrapEnv(this.env, requestIdFromEventArg(request))); + fetch(request) { + return withRequestContext(request, () => notifyWorkflowCallback(request, wrapEnv(this.env))); } } @@ -105,67 +204,43 @@ const WORKFLOW_BINDINGS = ${workflowBindingJson}; const DO_BACKEND_BINDING = "__WDL_DO_BACKEND__"; const DO_OWNER_NETWORK_BINDING = "__WDL_DO_OWNER_NETWORK__"; const WORKFLOWS_BACKEND_BINDING = "__WDL_WORKFLOWS_BACKEND__"; -const HOST_BINDINGS_WRAPPED = Symbol("wdl.host-bindings-wrapped"); +const HOST_BINDINGS_WRAPPED = __WdlHostRuntime__.createPrivateSymbol("wdl.host-bindings-wrapped"); const INTERNAL_BINDING_RE = /^__WDL_[A-Za-z0-9_]*__$/; +const REQUEST_CONTEXT = __WdlHostRuntime__.createRequestContext(); function requestIdFromEventArg(arg) { - if (!arg || !arg.headers || typeof arg.headers.get !== "function") return null; - return arg.headers.get("x-request-id"); -} - -function createRequestContext(requestId = null) { - // workerd constructs a fresh WorkerEntrypoint instance per call; covered by - // service-bindings-rpc.test.js. Do not share this object across instances. - return { requestId }; + return __WdlHostRuntime__.requestIdFromEventArg(arg); } -function requestIdOptions(requestIdOrContext) { - if (requestIdOrContext && typeof requestIdOrContext === "object") { - return { requestIdProvider: () => requestIdOrContext.requestId }; - } - return { requestId: requestIdOrContext }; +function requestIdOptions() { + return { requestIdProvider: REQUEST_CONTEXT.currentRequestId }; } -function doOptions(requestIdOrContext, backend, ownerNetwork) { - return { ...requestIdOptions(requestIdOrContext), backend, ownerNetwork }; +function doOptions(backend, ownerNetwork) { + return { ...requestIdOptions(), backend, ownerNetwork }; } -function workflowOptions(requestIdOrContext, backend) { - return { ...requestIdOptions(requestIdOrContext), backend }; +function workflowOptions(backend) { + return { ...requestIdOptions(), backend }; } -function withRequestContext(context, arg, fn) { - const previous = context.requestId; - const requestId = requestIdFromEventArg(arg); - if (requestId) context.requestId = requestId; - try { - const result = fn(); - if (result && typeof result.then === "function") { - return Promise.resolve(result).finally(() => { - context.requestId = previous; - }); - } - context.requestId = previous; - return result; - } catch (err) { - context.requestId = previous; - throw err; - } +function withRequestContext(arg, fn) { + return REQUEST_CONTEXT.runWithRequestContext(requestIdFromEventArg(arg), fn); } -function wrapClassInstance(instance, requestContext) { - return new Proxy(instance, { - get(target, prop, receiver) { - const value = Reflect.get(target, prop, receiver); +function wrapClassInstance(instance) { + return __WdlHostRuntime__.createProxy(instance, { + get(target, prop) { + const value = __WdlHostRuntime__.reflectGet(target, prop, target); if (typeof value !== "function") return value; return function(...args) { - return withRequestContext(requestContext, args[0], () => value.apply(target, args)); + return withRequestContext(args[0], () => __WdlHostRuntime__.applyFunction(value, target, args)); }; }, }); } -function wrapEnv(env, requestIdOrContext = null) { +function wrapEnv(env) { // Idempotence is a contract, not an optimization: WorkerEntrypoint methods // and default handlers may re-enter with an env already wrapped by this // module. A symbol marker cannot be forged by tenant vars/secrets. @@ -177,26 +252,26 @@ function wrapEnv(env, requestIdOrContext = null) { delete out[DO_BACKEND_BINDING]; delete out[DO_OWNER_NETWORK_BINDING]; delete out[WORKFLOWS_BACKEND_BINDING]; - for (const name of Object.keys(out)) { - if (INTERNAL_BINDING_RE.test(name)) delete out[name]; - } - for (const name of D1_BINDINGS) { - if (out[name] !== undefined) out[name] = new D1Database(out[name], requestIdOptions(requestIdOrContext)); - } - for (const name of R2_BINDINGS) { - if (out[name] !== undefined) out[name] = new R2Bucket(out[name], requestIdOptions(requestIdOrContext)); - } - for (const name of DO_BINDINGS) { + __WdlHostRuntime__.forEachArray(__WdlHostRuntime__.objectKeys(out), (name) => { + if (__WdlHostRuntime__.regexpTest(INTERNAL_BINDING_RE, name)) delete out[name]; + }); + __WdlHostRuntime__.forEachArray(D1_BINDINGS, (name) => { + if (out[name] !== undefined) out[name] = new D1Database(out[name], requestIdOptions()); + }); + __WdlHostRuntime__.forEachArray(R2_BINDINGS, (name) => { + if (out[name] !== undefined) out[name] = new R2Bucket(out[name], requestIdOptions()); + }); + __WdlHostRuntime__.forEachArray(DO_BINDINGS, (name) => { if (out[name] !== undefined) { - out[name] = new DurableObjectNamespace(out[name], doOptions(requestIdOrContext, doBackend, doOwnerNetwork)); + out[name] = new DurableObjectNamespace(out[name], doOptions(doBackend, doOwnerNetwork)); } - } - for (const [name, metadata] of Object.entries(WORKFLOW_BINDINGS)) { + }); + __WdlHostRuntime__.forEachObjectEntry(WORKFLOW_BINDINGS, (name, metadata) => { if (out[name] !== undefined) { - out[name] = new Workflow(out[name] || metadata, workflowOptions(requestIdOrContext, workflowsBackend)); + out[name] = new Workflow(out[name] || metadata, workflowOptions(workflowsBackend)); } - } - Object.defineProperty(out, HOST_BINDINGS_WRAPPED, { value: true }); + }); + __WdlHostRuntime__.defineProperty(out, HOST_BINDINGS_WRAPPED, { value: true }); return out; } @@ -246,13 +321,13 @@ async function notifyWorkflowCallback(request, env) { function wrapHandler(owner, fn) { return function(arg1, env, ctx) { - return fn.call(owner, arg1, wrapEnv(env, requestIdFromEventArg(arg1)), ctx); + return withRequestContext(arg1, () => __WdlHostRuntime__.applyFunction(fn, owner, [arg1, wrapEnv(env), ctx])); }; } const HOST_WRAPPED_HANDLER_KEYS = ["fetch", "scheduled", "queue", "tail"]; -const raw = user.default; +const raw = __WdlUserModule__.default; let wrappedDefault = raw; if (raw && typeof raw === "object") { @@ -260,20 +335,24 @@ if (raw && typeof raw === "object") { const wrapDefaultFunctionKey = (key) => { const fn = raw[key]; if (typeof fn === "function") { - wrappedDefault[key] = wrapHandler(raw, fn); + __WdlHostRuntime__.defineProperty(wrappedDefault, key, { + value: wrapHandler(raw, fn), + writable: true, + enumerable: true, + configurable: true, + }); } }; - for (const key of HOST_WRAPPED_HANDLER_KEYS) { + __WdlHostRuntime__.forEachArray(HOST_WRAPPED_HANDLER_KEYS, (key) => { wrapDefaultFunctionKey(key); - } + }); } else if (typeof raw === "function") { - const source = Function.prototype.toString.call(raw); - if (/^\\s*class\\b/.test(source)) { + const source = __WdlHostRuntime__.functionSource(raw); + if (__WdlHostRuntime__.regexpTest(/^\\s*class\\b/, source)) { wrappedDefault = class extends raw { constructor(ctx, env) { - const requestContext = createRequestContext(); - super(ctx, wrapEnv(env, requestContext)); - return wrapClassInstance(this, requestContext); + super(ctx, wrapEnv(env)); + return wrapClassInstance(this); } }; } else { diff --git a/runtime/r2-client.js b/runtime/r2-client.js index 83e4e72..5de68be 100644 --- a/runtime/r2-client.js +++ b/runtime/r2-client.js @@ -1,8 +1,11 @@ import { + R2_HTTP_METADATA_FIELDS, R2_OBJECT_MAX_BUFFER_BYTES, assertR2BufferSize, normalizeR2ListLimit, normalizeR2ObjectKey, + r2CacheExpiryFromHeaders, + setR2CacheExpiryHeader, } from "./_wdl-r2-utils.js"; import { requestIdFromOptions } from "./_wdl-request-id.js"; @@ -19,6 +22,9 @@ import { requestIdFromOptions } from "./_wdl-request-id.js"; */ const utf8Encoder = new TextEncoder(); const utf8Decoder = new TextDecoder(); +const intrinsicReflectApply = Reflect.apply; +const intrinsicWeakMapGet = WeakMap.prototype.get; +const intrinsicWeakMapSet = WeakMap.prototype.set; /** @param {unknown} value */ function dateFromUnknown(value) { @@ -40,6 +46,13 @@ function bytesToArrayBuffer(bytes) { ); } +/** @param {ReadableStreamDefaultReader} reader @param {unknown} reason */ +function cancelReaderBestEffort(reader, reason) { + try { + void reader.cancel(reason).catch(() => {}); + } catch {} +} + /** @param {ReadableStream} stream @param {string} operation */ async function readStreamWithLimit(stream, operation) { const reader = stream.getReader(); @@ -53,9 +66,10 @@ async function readStreamWithLimit(stream, operation) { const chunk = value instanceof Uint8Array ? value : new Uint8Array(value); total += chunk.byteLength; if (total > R2_OBJECT_MAX_BUFFER_BYTES) { - await reader.cancel( + cancelReaderBestEffort( + reader, `R2 ${operation}: object exceeds ${R2_OBJECT_MAX_BUFFER_BYTES} byte limit` - ).catch(() => {}); + ); assertR2BufferSize(total, operation); } chunks.push(chunk); @@ -94,9 +108,10 @@ function cappedReadableStream(stream, operation) { const chunk = value instanceof Uint8Array ? value : new Uint8Array(value); total += chunk.byteLength; if (total > R2_OBJECT_MAX_BUFFER_BYTES) { - await reader.cancel( + cancelReaderBestEffort( + reader, `R2 ${operation}: object exceeds ${R2_OBJECT_MAX_BUFFER_BYTES} byte limit` - ).catch(() => {}); + ); try { reader.releaseLock(); } catch {} try { assertR2BufferSize(total, operation); @@ -117,37 +132,33 @@ function cappedReadableStream(stream, operation) { }); } -/** @param {Headers} headers */ -function headersToHttpMetadata(headers) { - const expires = headers.get("expires"); - return { - contentType: headers.get("content-type") || undefined, - contentLanguage: headers.get("content-language") || undefined, - contentDisposition: headers.get("content-disposition") || undefined, - contentEncoding: headers.get("content-encoding") || undefined, - cacheControl: headers.get("cache-control") || undefined, - cacheExpiry: expires ? new Date(expires).getTime() : undefined, - }; +/** @param {Headers} headers @param {{ strictExpiry?: boolean }} [options] */ +function headersToHttpMetadata(headers, { strictExpiry = false } = {}) { + /** @type {AnyRecord} */ + const out = {}; + for (const [field, header] of R2_HTTP_METADATA_FIELDS) { + out[field] = headers.get(header) || undefined; + } + const cacheExpiry = r2CacheExpiryFromHeaders(headers, { canonical: strictExpiry }); + if (strictExpiry && headers.has("expires") && cacheExpiry === undefined) { + throw new TypeError("R2 httpMetadata Expires header must be canonical IMF-fixdate"); + } + out.cacheExpiry = cacheExpiry; + return out; } /** @param {unknown} input */ function normalizeHttpMetadata(input) { if (input == null) return undefined; - if (input instanceof Headers) return headersToHttpMetadata(input); + if (input instanceof Headers) return headersToHttpMetadata(input, { strictExpiry: true }); if (typeof input !== "object" || Array.isArray(input)) { throw new TypeError("R2 httpMetadata must be an object or Headers"); } /** @type {AnyRecord} */ const out = {}; const record = /** @type {AnyRecord} */ (input); - for (const key of [ - "contentType", - "contentLanguage", - "contentDisposition", - "contentEncoding", - "cacheControl", - ]) { - if (record[key] != null) out[key] = String(record[key]); + for (const [field] of R2_HTTP_METADATA_FIELDS) { + if (record[field] != null) out[field] = String(record[field]); } if (record.cacheExpiry != null) { const ms = record.cacheExpiry instanceof Date @@ -280,9 +291,19 @@ function metaWithDates(meta) { /** @type {WeakMap} */ const bucketState = new WeakMap(); +/** @param {WeakMap} map @param {object} key */ +function weakMapGet(map, key) { + return intrinsicReflectApply(intrinsicWeakMapGet, map, [key]); +} + +/** @param {WeakMap} map @param {object} key @param {unknown} value */ +function weakMapSet(map, key, value) { + intrinsicReflectApply(intrinsicWeakMapSet, map, [key, value]); +} + /** @param {R2Bucket} bucket */ function bucketRequestMeta(bucket) { - const state = bucketState.get(bucket); + const state = /** @type {R2BucketState | undefined} */ (weakMapGet(bucketState, bucket)); if (!state) return {}; const requestId = requestIdFromOptions(state.requestIdOptions); return requestId ? { requestId } : {}; @@ -320,17 +341,11 @@ export class R2Object { /** @param {Headers} headers */ writeHttpMetadata(headers) { const h = this.httpMetadata || {}; - const contentType = stringOrUndefined(h.contentType); - const contentLanguage = stringOrUndefined(h.contentLanguage); - const contentDisposition = stringOrUndefined(h.contentDisposition); - const contentEncoding = stringOrUndefined(h.contentEncoding); - const cacheControl = stringOrUndefined(h.cacheControl); - if (contentType) headers.set("content-type", contentType); - if (contentLanguage) headers.set("content-language", contentLanguage); - if (contentDisposition) headers.set("content-disposition", contentDisposition); - if (contentEncoding) headers.set("content-encoding", contentEncoding); - if (cacheControl) headers.set("cache-control", cacheControl); - if (h.cacheExpiry) headers.set("expires", dateFromUnknown(h.cacheExpiry).toUTCString()); + for (const [field, header] of R2_HTTP_METADATA_FIELDS) { + const value = stringOrUndefined(h[field]); + if (value) headers.set(header, value); + } + setR2CacheExpiryHeader(headers, h.cacheExpiry); } } @@ -405,7 +420,7 @@ export class R2Bucket { constructor(stub, options = {}) { // Provider is used by class-style entrypoints where the same env wrapper // can serve different fetch/queue/scheduled invocations with different ids. - bucketState.set(this, { + weakMapSet(bucketState, this, { stub, requestIdOptions: options, }); @@ -413,7 +428,7 @@ export class R2Bucket { /** @param {string} key */ async head(key) { - const { stub } = /** @type {R2BucketState} */ (bucketState.get(this)); + const { stub } = /** @type {R2BucketState} */ (weakMapGet(bucketState, this)); if (typeof stub.head !== "function") throw new TypeError("R2 stub head is not configured"); const meta = await stub.head(normalizeR2ObjectKey(key), bucketRequestMeta(this)); return meta ? new R2Object(meta) : null; @@ -421,7 +436,7 @@ export class R2Bucket { /** @param {string} key @param {AnyRecord} [options] */ async get(key, options) { - const { stub } = /** @type {R2BucketState} */ (bucketState.get(this)); + const { stub } = /** @type {R2BucketState} */ (weakMapGet(bucketState, this)); if (typeof stub.get !== "function") throw new TypeError("R2 stub get is not configured"); const result = await stub.get( normalizeR2ObjectKey(key), @@ -435,14 +450,15 @@ export class R2Bucket { /** @param {string} key @param {unknown} value @param {AnyRecord} [options] */ async put(key, value, options) { + const normalizedOptions = normalizePutOptions(options); const bytes = await valueToBytes(value); assertR2BufferSize(bytes.byteLength, "put"); - const { stub } = /** @type {R2BucketState} */ (bucketState.get(this)); + const { stub } = /** @type {R2BucketState} */ (weakMapGet(bucketState, this)); if (typeof stub.put !== "function") throw new TypeError("R2 stub put is not configured"); const meta = await stub.put( normalizeR2ObjectKey(key), bytes, - normalizePutOptions(options), + normalizedOptions, bucketRequestMeta(this) ); return meta ? new R2Object(meta) : null; @@ -460,7 +476,7 @@ export class R2Bucket { /** @param {string | string[]} keys */ async delete(keys) { - const { stub } = /** @type {R2BucketState} */ (bucketState.get(this)); + const { stub } = /** @type {R2BucketState} */ (weakMapGet(bucketState, this)); if (typeof stub.delete !== "function") throw new TypeError("R2 stub delete is not configured"); if (Array.isArray(keys)) { await stub.delete( @@ -474,7 +490,7 @@ export class R2Bucket { /** @param {AnyRecord} [options] */ async list(options = {}) { - const { stub } = /** @type {R2BucketState} */ (bucketState.get(this)); + const { stub } = /** @type {R2BucketState} */ (weakMapGet(bucketState, this)); if (typeof stub.list !== "function") throw new TypeError("R2 stub list is not configured"); const out = await stub.list({ prefix: options.prefix == null ? undefined : String(options.prefix), diff --git a/runtime/r2-utils.js b/runtime/r2-utils.js index d029ce4..d25aeca 100644 --- a/runtime/r2-utils.js +++ b/runtime/r2-utils.js @@ -3,6 +3,45 @@ export const R2_LIST_LIMIT_MAX = 1000; // Keep in sync with shared/ns-pattern.js. This file is embedded into loaded // workers as _wdl-r2-utils.js, so it must stay standalone. export const R2_BUCKET_NAME_RE = /^[a-z0-9][a-z0-9-]{0,62}$/; +export const R2_HTTP_METADATA_FIELDS = Object.freeze([ + Object.freeze(["contentType", "content-type"]), + Object.freeze(["contentLanguage", "content-language"]), + Object.freeze(["contentDisposition", "content-disposition"]), + Object.freeze(["contentEncoding", "content-encoding"]), + Object.freeze(["cacheControl", "cache-control"]), +]); +const R2_IMF_FIXDATE_RE = /^(?:Mon|Tue|Wed|Thu|Fri|Sat|Sun), \d{2} (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \d{4} \d{2}:\d{2}:\d{2} GMT$/; + +/** @param {Headers} headers @param {{ canonical?: boolean }} [options] */ +export function r2CacheExpiryFromHeaders(headers, { canonical = false } = {}) { + const expires = headers.get("expires"); + if (!expires) return undefined; + const ms = new Date(expires).getTime(); + if (canonical && ( + !R2_IMF_FIXDATE_RE.test(expires) || + !Number.isFinite(ms) || + new Date(ms).toUTCString() !== expires + )) return undefined; + return Number.isFinite(ms) ? ms : undefined; +} + +/** @param {Headers} headers @param {unknown} value */ +export function setR2CacheExpiryHeader(headers, value) { + if (value == null) return; + let date; + if (value instanceof Date || typeof value === "number") { + date = new Date(value); + } else if (typeof value === "string" && value.trim() !== "") { + const timestamp = Number(value); + date = new Date(Number.isFinite(timestamp) ? timestamp : value); + } else { + throw new TypeError("R2 httpMetadata.cacheExpiry must be a valid Date or timestamp"); + } + if (!Number.isFinite(date.getTime())) { + throw new TypeError("R2 httpMetadata.cacheExpiry must be a valid Date or timestamp"); + } + headers.set("expires", date.toUTCString()); +} /** * @param {unknown} bucketName diff --git a/runtime/runtime.js b/runtime/runtime.js index 74ae686..29147cf 100644 --- a/runtime/runtime.js +++ b/runtime/runtime.js @@ -1,7 +1,7 @@ // Runtime helpers for user-runtime and system-runtime. This module owns // service-name binding, logger setup, metrics access, request-scope creation, // and the per-process loaded-worker registry that drives historical-version -// eviction; runtime/index.js owns loader dispatch and worker event handling. +// eviction; runtime/load.js owns the shared workerLoader cold-load wrapper. import { createLogLevelBinder, diff --git a/runtime/tail-forwarder.js b/runtime/tail-forwarder.js index 00f7eaf..01f0fc7 100644 --- a/runtime/tail-forwarder.js +++ b/runtime/tail-forwarder.js @@ -6,6 +6,7 @@ import { optionalRedisProxyBaseUrl, proxyEndpoint, } from "runtime-bindings-proxy"; +import { errorMessage } from "shared-errors"; import { withInternalAuth } from "shared-internal-auth"; // Active-set cache. Keep the timing constants beside the implementation; @@ -298,3 +299,51 @@ export function emitRuntimeTailEvent({ env, ctx, identity, event, phase, fields ctx.waitUntil(task); return task; } + +/** + * @param {{ + * env: RuntimeTailEnv, + * ctx: { waitUntil?: (promise: Promise) => void }, + * identity: { namespace?: string, workerName?: string, workerId?: string, requestId?: string | null }, + * event: string, + * fields: Record, + * }} options + */ +export function startTailEnvelope({ env, ctx, identity, event, fields }) { + const startedAt = Date.now(); + const startTailEvent = emitRuntimeTailEvent({ + env, ctx, identity, + event, + phase: "start", + fields, + }); + + /** @param {Record} extraFields */ + function finish(extraFields) { + const durationMs = Date.now() - startedAt; + emitRuntimeTailEvent({ + env, ctx, identity, + event, + phase: "finish", + after: startTailEvent, + fields: { + ...fields, + ...extraFields, + duration_ms: durationMs, + }, + }); + return durationMs; + } + + return { + finish, + /** @param {unknown} err @param {Record} [extraFields] */ + finishError(err, extraFields = {}) { + return finish({ + ...extraFields, + outcome: "error", + error: errorMessage(err), + }); + }, + }; +} diff --git a/runtime/workflows-client.js b/runtime/workflows-client.js index 8c17494..8315ea1 100644 --- a/runtime/workflows-client.js +++ b/runtime/workflows-client.js @@ -199,6 +199,7 @@ export class Workflow { throw new Error("Workflow backend is not configured"); } const metadata = this.#metadata; + const requestId = requestIdFromOptions(this.#options); const body = { ...fields, ns: metadata.ns, @@ -207,11 +208,14 @@ export class Workflow { workflowName: metadata.name, workflowKey: metadata.workflowKey, className: metadata.className, - requestId: requestIdFromOptions(this.#options), + requestId, }; const response = await this.#backend.fetch(`${WORKFLOWS_BASE_URL}/${endpoint}`, { method: "POST", - headers: { "content-type": "application/json" }, + headers: { + "content-type": "application/json", + ...(requestId ? { "x-request-id": requestId } : {}), + }, body: JSON.stringify(body), }); return await readWorkflowResponse(response); diff --git a/rust/common/Cargo.toml b/rust/common/Cargo.toml index 87a8aad..9c0ee7a 100644 --- a/rust/common/Cargo.toml +++ b/rust/common/Cargo.toml @@ -14,6 +14,7 @@ path = "src/lib.rs" [features] default = ["axum"] axum = ["dep:axum"] +test-support = [] [dependencies] axum = { workspace = true, features = ["http1"], optional = true } diff --git a/rust/common/src/internal_auth.rs b/rust/common/src/internal_auth.rs index c0f1407..d7e29ad 100644 --- a/rust/common/src/internal_auth.rs +++ b/rust/common/src/internal_auth.rs @@ -1,12 +1,17 @@ use std::env; #[cfg(feature = "axum")] -use axum::http::HeaderMap; +use axum::{ + http::{HeaderMap, StatusCode, header::CONTENT_TYPE}, + response::{IntoResponse, Response}, +}; use subtle::ConstantTimeEq; pub const INTERNAL_AUTH_HEADER: &str = "x-wdl-internal-auth"; pub const INTERNAL_AUTH_ENV: &str = "WDL_INTERNAL_AUTH_TOKEN"; pub const INTERNAL_AUTH_PREVIOUS_ENV: &str = "WDL_INTERNAL_AUTH_PREVIOUS_TOKEN"; +pub const INTERNAL_AUTH_FAILURE_CODE: &str = "internal_auth_failed"; +pub const INTERNAL_AUTH_FAILURE_MESSAGE: &str = "Internal authentication failed"; #[derive(Clone, Debug, Eq, PartialEq)] pub struct InternalAuthTokens { @@ -15,9 +20,13 @@ pub struct InternalAuthTokens { } fn validate_internal_auth_token(name: &str, token: String) -> Result { - if token.is_empty() || !token.is_ascii() { + if token.is_empty() + || token + .bytes() + .any(|byte| !(0x21..=0x7e).contains(&byte) || byte == b',') + { return Err(format!( - "{name} must be configured as a non-empty ASCII string" + "{name} must be configured as visible ASCII without whitespace or commas" )); } Ok(token) @@ -71,9 +80,12 @@ pub fn internal_auth_matches_any(actual: Option<&str>, expected: &InternalAuthTo #[cfg(feature = "axum")] pub fn internal_auth_header_value(headers: &HeaderMap) -> Option<&str> { - headers - .get(INTERNAL_AUTH_HEADER) - .and_then(|value| value.to_str().ok()) + let mut values = headers.get_all(INTERNAL_AUTH_HEADER).iter(); + let value = values.next()?.to_str().ok()?; + if values.next().is_some() { + return None; + } + Some(value) } #[cfg(feature = "axum")] @@ -81,10 +93,89 @@ pub fn internal_auth_headers_match(headers: &HeaderMap, expected: &InternalAuthT internal_auth_matches_any(internal_auth_header_value(headers), expected) } +#[cfg(feature = "axum")] +pub fn internal_auth_failure_response() -> Response { + let body = serde_json::json!({ + "error": INTERNAL_AUTH_FAILURE_CODE, + "message": INTERNAL_AUTH_FAILURE_MESSAGE, + }) + .to_string(); + ( + StatusCode::UNAUTHORIZED, + [(CONTENT_TYPE, "application/json")], + body, + ) + .into_response() +} + #[cfg(test)] mod tests { use super::*; + fn contract() -> serde_json::Value { + serde_json::from_str(include_str!( + "../../../tests/fixtures/internal-auth-contract.json" + )) + .expect("internal auth contract fixture") + } + + #[test] + fn internal_auth_literals_and_rotation_match_contract() { + let contract = contract(); + assert_eq!(contract["header"], INTERNAL_AUTH_HEADER); + assert_eq!(contract["currentEnv"], INTERNAL_AUTH_ENV); + assert_eq!(contract["previousEnv"], INTERNAL_AUTH_PREVIOUS_ENV); + assert_eq!(contract["failure"]["error"], INTERNAL_AUTH_FAILURE_CODE); + assert_eq!( + contract["failure"]["message"], + INTERNAL_AUTH_FAILURE_MESSAGE + ); + + for case in contract["tokenCases"].as_array().expect("tokenCases array") { + let value = case["value"].as_str().expect("token value").to_string(); + assert_eq!( + validate_internal_auth_token(INTERNAL_AUTH_ENV, value).is_ok(), + case["accepted"].as_bool().expect("accepted boolean") + ); + } + + for case in contract["rotationCases"] + .as_array() + .expect("rotationCases array") + { + let tokens = InternalAuthTokens { + current: case["current"].as_str().expect("current token").to_string(), + previous: case["previous"].as_str().map(str::to_string), + }; + assert_eq!( + internal_auth_matches_any(case["actual"].as_str(), &tokens), + case["accepted"].as_bool().expect("accepted boolean") + ); + } + + #[cfg(feature = "axum")] + for case in contract["headerCases"] + .as_array() + .expect("headerCases array") + { + let tokens = InternalAuthTokens { + current: case["current"].as_str().expect("current token").to_string(), + previous: case["previous"].as_str().map(str::to_string), + }; + let mut headers = HeaderMap::new(); + for value in case["values"].as_array().expect("header values") { + headers.append( + INTERNAL_AUTH_HEADER, + value.as_str().expect("header value").parse().unwrap(), + ); + } + assert_eq!( + internal_auth_headers_match(&headers, &tokens), + case["accepted"].as_bool().expect("accepted boolean") + ); + } + } + #[test] fn internal_auth_matches_requires_exact_token() { assert!(internal_auth_matches(Some("secret"), "secret")); @@ -127,12 +218,40 @@ mod tests { } #[test] - fn internal_auth_tokens_must_be_ascii() { + fn internal_auth_tokens_must_be_visible_ascii_without_commas() { assert_eq!( validate_internal_auth_token(INTERNAL_AUTH_ENV, "ascii-token".to_string()).unwrap(), "ascii-token" ); assert!(validate_internal_auth_token(INTERNAL_AUTH_ENV, "".to_string()).is_err()); assert!(validate_internal_auth_token(INTERNAL_AUTH_ENV, "tokén".to_string()).is_err()); + assert!( + validate_internal_auth_token(INTERNAL_AUTH_ENV, "token,other".to_string()).is_err() + ); + assert!(validate_internal_auth_token(INTERNAL_AUTH_ENV, " token".to_string()).is_err()); + assert!(validate_internal_auth_token(INTERNAL_AUTH_ENV, "token ".to_string()).is_err()); + assert!( + validate_internal_auth_token(INTERNAL_AUTH_ENV, "token\0value".to_string()).is_err() + ); + } + + #[cfg(feature = "axum")] + #[tokio::test] + async fn internal_auth_failure_response_matches_contract() { + let contract = contract(); + let response = internal_auth_failure_response(); + assert_eq!( + u64::from(response.status().as_u16()), + contract["failure"]["status"] + .as_u64() + .expect("failure status") + ); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .expect("internal auth failure body"); + let body: serde_json::Value = + serde_json::from_slice(&body).expect("internal auth failure JSON"); + assert_eq!(body["error"], contract["failure"]["error"]); + assert_eq!(body["message"], contract["failure"]["message"]); } } diff --git a/rust/common/src/lib.rs b/rust/common/src/lib.rs index bbc239b..cd2c5b0 100644 --- a/rust/common/src/lib.rs +++ b/rust/common/src/lib.rs @@ -5,8 +5,9 @@ //! health probes, shutdown/in-flight tracking, metric storage/formatting, FNV hashing, //! request-id sanitization, internal-auth constants/token matching, identity grammar, //! version and queue-key helpers, Redis connection and EVAL command helpers, time -//! helpers, structured error fields, and UTF-8-safe text helpers. It should not own -//! service protocols, Redis schemas, dispatch policy, or lifecycle behavior. +//! helpers, structured error fields, test-only process-environment overrides, and +//! UTF-8-safe text helpers. It should not own service protocols, Redis schemas, +//! dispatch policy, or lifecycle behavior. pub mod env; pub mod hash; @@ -21,12 +22,21 @@ pub mod redis_conn; pub mod redis_eval; pub mod request_id; pub mod shutdown; +#[cfg(any(test, feature = "test-support"))] +pub mod test_env; pub mod text; pub mod time; pub mod version; #[cfg(test)] pub(crate) mod test_fixtures { + pub(crate) fn observability_contract() -> serde_json::Value { + serde_json::from_str(include_str!( + "../../../tests/fixtures/observability-contract.json" + )) + .expect("observability contract fixture parses") + } + pub(crate) fn identity_cases(field: &str) -> Vec<(String, bool)> { let fixture: serde_json::Value = serde_json::from_str(include_str!( "../../../tests/fixtures/cross-language-identity.json" diff --git a/rust/common/src/log.rs b/rust/common/src/log.rs index 0adfec4..df484a2 100644 --- a/rust/common/src/log.rs +++ b/rust/common/src/log.rs @@ -2,6 +2,7 @@ use chrono::{SecondsFormat, Utc}; use serde_json::Value as JsonValue; #[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)] +#[repr(u8)] pub enum LogLevel { Debug = 10, Info = 20, @@ -9,6 +10,24 @@ pub enum LogLevel { Error = 40, } +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum LogStream { + Stdout, + Stderr, +} + +fn should_emit(level: LogLevel, min_level: LogLevel) -> bool { + level >= min_level +} + +fn stream_for_level(level: LogLevel) -> LogStream { + if level == LogLevel::Error { + LogStream::Stderr + } else { + LogStream::Stdout + } +} + impl LogLevel { pub fn parse(value: &str) -> Option { match value.to_ascii_lowercase().as_str() { @@ -45,14 +64,13 @@ pub fn emit_log_line( event: &str, fields: JsonValue, ) { - if level < min_level { + if !should_emit(level, min_level) { return; } let line = log_payload_line(service, level, event, fields); - if level >= LogLevel::Error { - eprintln!("{line}"); - } else { - println!("{line}"); + match stream_for_level(level) { + LogStream::Stdout => println!("{line}"), + LogStream::Stderr => eprintln!("{line}"), } } @@ -119,8 +137,16 @@ fn now_log_ts() -> String { #[cfg(test)] mod tests { use super::*; + use crate::test_fixtures::observability_contract; use serde_json::json; + fn timestamp_shape(value: &str) -> String { + value + .chars() + .map(|ch| if ch.is_ascii_digit() { '0' } else { ch }) + .collect() + } + #[test] fn level_parse_is_case_insensitive() { assert_eq!(LogLevel::parse("debug"), Some(LogLevel::Debug)); @@ -131,10 +157,50 @@ mod tests { } #[test] - fn level_ord_lets_us_gate() { - assert!(LogLevel::Debug < LogLevel::Info); - assert!(LogLevel::Info < LogLevel::Warn); - assert!(LogLevel::Warn < LogLevel::Error); + fn log_levels_gating_and_streams_match_cross_language_fixture() { + let contract = observability_contract(); + let levels = contract["logEnvelope"]["levels"] + .as_array() + .expect("logEnvelope.levels is an array"); + + for entry in levels { + let name = entry["name"].as_str().expect("log level name is a string"); + let priority = entry["priority"] + .as_u64() + .expect("log level priority is a number"); + let stream = entry["stream"] + .as_str() + .expect("log level stream is a string"); + let level = LogLevel::parse(name).expect("fixture log level is supported"); + let actual_stream = match stream_for_level(level) { + LogStream::Stdout => "stdout", + LogStream::Stderr => "stderr", + }; + assert_eq!(u64::from(level as u8), priority, "{name} priority"); + assert_eq!(actual_stream, stream, "{name} stream"); + } + + for min_entry in levels { + let min_name = min_entry["name"] + .as_str() + .expect("minimum log level name is a string"); + let min_priority = min_entry["priority"] + .as_u64() + .expect("minimum log level priority is a number"); + let min_level = LogLevel::parse(min_name).expect("minimum log level is supported"); + for entry in levels { + let name = entry["name"].as_str().expect("log level name is a string"); + let priority = entry["priority"] + .as_u64() + .expect("log level priority is a number"); + let level = LogLevel::parse(name).expect("fixture log level is supported"); + assert_eq!( + should_emit(level, min_level), + priority >= min_priority, + "{name} at minimum {min_name}" + ); + } + } } #[test] @@ -159,6 +225,10 @@ mod tests { #[test] fn log_payload_line_matches_js_envelope_order() { + let contract = observability_contract(); + let ordered_keys = contract["logEnvelope"]["orderedKeys"] + .as_array() + .expect("logEnvelope.orderedKeys is an array"); let line = log_payload_line( "scheduler", LogLevel::Info, @@ -166,25 +236,24 @@ mod tests { json!({ "port": 7070 }), ); - assert!( - line.starts_with(r#"{"ts":"#), - "Rust structured logs should match JS envelope order: {line}" - ); - assert!(line.contains(r#","service":"scheduler","level":"info","event":"started","#)); + let mut position = 0; + for (index, key) in ordered_keys.iter().enumerate() { + let key = key.as_str().expect("ordered log key is a string"); + let marker = format!("{}\"{key}\":", if index == 0 { "{" } else { "," }); + let offset = line[position..] + .find(&marker) + .unwrap_or_else(|| panic!("missing ordered marker {marker:?} in {line}")); + position += offset + marker.len(); + } } #[test] fn log_timestamp_matches_js_iso_millisecond_shape() { + let contract = observability_contract(); + let expected = contract["logEnvelope"]["timestampShape"] + .as_str() + .expect("logEnvelope.timestampShape is a string"); let ts = now_log_ts(); - assert_eq!(ts.len(), "2026-01-01T00:00:00.000Z".len()); - assert_eq!(&ts[4..5], "-"); - assert_eq!(&ts[7..8], "-"); - assert_eq!(&ts[10..11], "T"); - assert_eq!(&ts[13..14], ":"); - assert_eq!(&ts[16..17], ":"); - assert_eq!(&ts[19..20], "."); - assert_eq!(&ts[23..24], "Z"); - assert!(ts[20..23].chars().all(|ch| ch.is_ascii_digit())); - assert!(!ts.contains("+00:00")); + assert_eq!(timestamp_shape(&ts), expected); } } diff --git a/rust/common/src/metrics.rs b/rust/common/src/metrics.rs index b316acc..5986d36 100644 --- a/rust/common/src/metrics.rs +++ b/rust/common/src/metrics.rs @@ -348,16 +348,10 @@ fn summary_sample_cmp(left: &SummarySample, right: &SummarySample) -> std::cmp:: #[cfg(test)] mod tests { use super::*; + use crate::test_fixtures::observability_contract; static PANIC_HOOK_LOCK: Mutex<()> = Mutex::new(()); - fn observability_contract() -> serde_json::Value { - serde_json::from_str(include_str!( - "../../../tests/fixtures/observability-contract.json" - )) - .expect("observability contract fixture parses") - } - #[test] fn metrics_contract_matches_cross_language_fixture() { let contract = observability_contract(); diff --git a/rust/common/src/queue_keys.rs b/rust/common/src/queue_keys.rs index ac11571..8c95c67 100644 --- a/rust/common/src/queue_keys.rs +++ b/rust/common/src/queue_keys.rs @@ -1,10 +1,16 @@ //! Queue Redis key helpers shared by producer and scheduler services. -use crate::identity::is_valid_runtime_load_ns; +use crate::identity::{is_valid_route_ns, is_valid_runtime_load_ns}; pub const QUEUE_CONSUMER_INDEX_KEY: &str = "queue:index:consumers"; pub const QUEUE_STREAM_INDEX_KEY: &str = "queue:index:streams"; pub const QUEUE_DELAYED_INDEX_KEY: &str = "queue:index:delayed"; +pub const QUEUE_CONSUMER_SCAN_PATTERN: &str = "queue-consumer:*:*"; +pub const QUEUE_STREAM_SCAN_PATTERN: &str = "queue:*:*:s"; +pub const QUEUE_DELAYED_SCAN_PATTERN: &str = "queue-delayed:*:*"; +pub const QUEUE_DELAYED_WAKE_STREAM: &str = "queue-delayed-wake"; +pub const QUEUE_DELAYED_WAKE_KEY_FIELD: &str = "delayed_key"; +pub const QUEUE_DELAYED_WAKE_VISIBLE_AT_FIELD: &str = "visible_at"; pub fn is_valid_queue_name(queue: &str) -> bool { let bytes = queue.as_bytes(); @@ -42,22 +48,22 @@ pub fn queue_consumer_key(ns: &str, queue: &str) -> String { pub fn parse_stream_key(key: &str) -> Option<(String, String)> { let rest = key.strip_prefix("queue:")?; let rest = rest.strip_suffix(":s")?; - split_queue_key_rest(rest) + split_queue_key_rest(rest, is_valid_runtime_load_ns) } pub fn parse_delayed_key(key: &str) -> Option<(String, String)> { let rest = key.strip_prefix("queue-delayed:")?; - split_queue_key_rest(rest) + split_queue_key_rest(rest, is_valid_runtime_load_ns) } pub fn parse_consumer_key(key: &str) -> Option<(String, String)> { let rest = key.strip_prefix("queue-consumer:")?; - split_queue_key_rest(rest) + split_queue_key_rest(rest, is_valid_route_ns) } -fn split_queue_key_rest(rest: &str) -> Option<(String, String)> { +fn split_queue_key_rest(rest: &str, is_valid_ns: fn(&str) -> bool) -> Option<(String, String)> { let (ns, queue) = rest.split_once(':')?; - if !is_valid_runtime_load_ns(ns) || !is_valid_queue_name(queue) { + if !is_valid_ns(ns) || !is_valid_queue_name(queue) { return None; } Some((ns.to_string(), queue.to_string())) @@ -67,6 +73,25 @@ fn split_queue_key_rest(rest: &str) -> Option<(String, String)> { mod tests { use super::*; use crate::test_fixtures::identity_cases; + use serde_json::{Value as JsonValue, json}; + + fn assert_queue_key_parse_contract(kind: &str, parse: fn(&str) -> Option<(String, String)>) { + let fixture: JsonValue = + serde_json::from_str(include_str!("../../../tests/fixtures/queue-key-parse.json")) + .expect("queue key parse fixture parses"); + for entry in fixture[kind] + .as_array() + .expect("queue key parse fixture field is an array") + { + let key = entry["key"] + .as_str() + .expect("queue key parse fixture key is a string"); + let actual = parse(key) + .map(|(ns, queue)| json!({ "ns": ns, "queue": queue })) + .unwrap_or(JsonValue::Null); + assert_eq!(actual, entry["parsed"], "{kind}:{key:?}"); + } + } #[test] fn queue_name_grammar_matches_cross_language_fixture() { @@ -80,6 +105,12 @@ mod tests { assert_eq!(QUEUE_CONSUMER_INDEX_KEY, "queue:index:consumers"); assert_eq!(QUEUE_STREAM_INDEX_KEY, "queue:index:streams"); assert_eq!(QUEUE_DELAYED_INDEX_KEY, "queue:index:delayed"); + assert_eq!(QUEUE_CONSUMER_SCAN_PATTERN, "queue-consumer:*:*"); + assert_eq!(QUEUE_STREAM_SCAN_PATTERN, "queue:*:*:s"); + assert_eq!(QUEUE_DELAYED_SCAN_PATTERN, "queue-delayed:*:*"); + assert_eq!(QUEUE_DELAYED_WAKE_STREAM, "queue-delayed-wake"); + assert_eq!(QUEUE_DELAYED_WAKE_KEY_FIELD, "delayed_key"); + assert_eq!(QUEUE_DELAYED_WAKE_VISIBLE_AT_FIELD, "visible_at"); assert_eq!(queue_stream_key("demo", "jobs"), "queue:demo:jobs:s"); assert_eq!(queue_delayed_key("demo", "jobs"), "queue-delayed:demo:jobs"); assert_eq!(queue_dlq_key("demo", "jobs"), "queue:demo:jobs:dlq"); @@ -106,41 +137,9 @@ mod tests { } #[test] - fn parse_queue_keys() { - assert_eq!( - parse_stream_key("queue:demo:jobs:s"), - Some(("demo".to_string(), "jobs".to_string())) - ); - assert_eq!( - parse_stream_key("queue:__platform__:jobs:s"), - Some(("__platform__".to_string(), "jobs".to_string())) - ); - assert_eq!( - parse_delayed_key("queue-delayed:demo:jobs"), - Some(("demo".to_string(), "jobs".to_string())) - ); - assert_eq!( - parse_consumer_key("queue-consumer:demo:jobs"), - Some(("demo".to_string(), "jobs".to_string())) - ); - assert_eq!( - parse_stream_key("queue:t:my-queue-1:s"), - Some(("t".to_string(), "my-queue-1".to_string())) - ); - assert_eq!(parse_stream_key("queue:demo:jobs"), None); - assert_eq!(parse_stream_key("queue-delayed:demo:jobs"), None); - assert_eq!(parse_stream_key("queue::jobs:s"), None); - assert_eq!(parse_stream_key("queue:admin:jobs:s"), None); - assert_eq!(parse_stream_key("queue:__community__:jobs:s"), None); - assert_eq!(parse_stream_key("queue:demo:Jobs:s"), None); - assert_eq!(parse_stream_key("queue:demo:jobs:extra:s"), None); - assert_eq!(parse_stream_key("notaqueue:demo:jobs:s"), None); - assert_eq!(parse_stream_key(""), None); - assert_eq!(parse_delayed_key("queue:demo:jobs:s"), None); - assert_eq!(parse_delayed_key("queue-delayed::jobs"), None); - assert_eq!(parse_delayed_key("queue-delayed:demo:"), None); - assert_eq!(parse_consumer_key("queue-consumer:demo"), None); - assert_eq!(parse_consumer_key("queue-consumer:"), None); - assert_eq!(parse_consumer_key("other:demo:jobs"), None); + fn queue_key_parsers_match_cross_language_fixture() { + assert_queue_key_parse_contract("stream", parse_stream_key); + assert_queue_key_parse_contract("delayed", parse_delayed_key); + assert_queue_key_parse_contract("consumer", parse_consumer_key); } } diff --git a/rust/common/src/request_id.rs b/rust/common/src/request_id.rs index a3ebeb9..7de1db5 100644 --- a/rust/common/src/request_id.rs +++ b/rust/common/src/request_id.rs @@ -1,21 +1,34 @@ +#[cfg(feature = "axum")] +use axum::http::HeaderMap; + +#[cfg(feature = "axum")] +const REQUEST_ID_HEADER: &str = "x-request-id"; + pub fn sanitize_request_id(raw: &str) -> Option { - let first = raw.split(',').next()?.trim(); + let first = raw + .split(',') + .next()? + .trim_matches(|ch: char| ch.is_ascii_whitespace()); if first.is_empty() || first.len() > 128 { return None; } - if first.chars().any(|ch| { - ch.is_ascii_whitespace() - || ch == '"' - || ch == '\\' - || (ch as u32) < 0x20 - || ch == '\u{7f}' - || ('\u{80}'..='\u{9f}').contains(&ch) - }) { + if first + .bytes() + .any(|byte| !(0x21..=0x7e).contains(&byte) || byte == b'"' || byte == b'\\') + { return None; } Some(first.to_string()) } +#[cfg(feature = "axum")] +pub fn request_id_from_headers(headers: &HeaderMap) -> Option { + headers + .get(REQUEST_ID_HEADER) + .and_then(|value| value.to_str().ok()) + .and_then(sanitize_request_id) +} + #[cfg(test)] mod tests { use super::*; @@ -44,6 +57,21 @@ mod tests { assert_eq!(sanitize_request_id("bad\"id"), None); assert_eq!(sanitize_request_id("bad\\id"), None); assert_eq!(sanitize_request_id("bad\nid"), None); + assert_eq!(sanitize_request_id("café"), None); + assert_eq!(sanitize_request_id("\u{85}rid"), None); + } + + #[cfg(feature = "axum")] + #[test] + fn request_id_header_adapter_rejects_non_ascii_wire_values() { + use axum::http::HeaderValue; + + let mut headers = HeaderMap::new(); + headers.insert( + REQUEST_ID_HEADER, + HeaderValue::from_bytes("é".as_bytes()).expect("obs-text header value"), + ); + assert_eq!(request_id_from_headers(&headers), None); } #[test] diff --git a/rust/common/src/test_env.rs b/rust/common/src/test_env.rs new file mode 100644 index 0000000..994a335 --- /dev/null +++ b/rust/common/src/test_env.rs @@ -0,0 +1,73 @@ +//! Serialized process-environment overrides for Rust tests. + +use std::ffi::OsString; +use std::sync::Mutex; + +static ENV_LOCK: Mutex<()> = Mutex::new(()); + +struct EnvRestore(Vec<(String, Option)>); + +impl Drop for EnvRestore { + fn drop(&mut self) { + for (key, value) in self.0.iter().rev() { + // SAFETY: ENV_LOCK remains held until after this restore guard drops. + match value { + Some(value) => unsafe { std::env::set_var(key, value) }, + None => unsafe { std::env::remove_var(key) }, + } + } + } +} + +pub fn with_temp_env(key: &str, value: Option<&str>, f: impl FnOnce() -> R) -> R { + with_temp_envs(&[(key, value)], f) +} + +pub fn with_temp_envs(items: &[(&str, Option<&str>)], f: impl FnOnce() -> R) -> R { + let _lock = ENV_LOCK + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + let _restore = EnvRestore( + items + .iter() + .map(|(key, _)| ((*key).to_string(), std::env::var_os(key))) + .collect(), + ); + for (key, value) in items { + // SAFETY: callers keep all environment reads inside the closure, and every + // WDL test environment override uses this crate-wide lock. + match value { + Some(value) => unsafe { std::env::set_var(key, value) }, + None => unsafe { std::env::remove_var(key) }, + } + } + f() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn restores_multiple_values_after_unwind_and_recovers_poisoned_lock() { + const FIRST: &str = "WDL_COMMON_TEST_ENV_FIRST"; + const SECOND: &str = "WDL_COMMON_TEST_ENV_SECOND"; + let before = [std::env::var_os(FIRST), std::env::var_os(SECOND)]; + + let result = std::panic::catch_unwind(|| { + with_temp_envs(&[(FIRST, Some("one")), (SECOND, None)], || { + assert_eq!(std::env::var(FIRST).as_deref(), Ok("one")); + assert_eq!(std::env::var_os(SECOND), None); + panic!("exercise panic-safe restore"); + }); + }); + assert!(result.is_err()); + assert_eq!(std::env::var_os(FIRST), before[0]); + assert_eq!(std::env::var_os(SECOND), before[1]); + + with_temp_env(FIRST, Some("two"), || { + assert_eq!(std::env::var(FIRST).as_deref(), Ok("two")); + }); + assert_eq!(std::env::var_os(FIRST), before[0]); + } +} diff --git a/rust/common/src/version.rs b/rust/common/src/version.rs index 2780690..3d32901 100644 --- a/rust/common/src/version.rs +++ b/rust/common/src/version.rs @@ -1,17 +1,20 @@ //! Worker version and bundle-key helpers shared by Rust services. //! //! JavaScript control code owns the canonical version tag grammar in -//! `shared/version.js`: `v[1-9][0-9]*`. Rust services that read bundle hashes +//! `shared/version.js`: `v[1-9][0-9]*`, bounded to JavaScript's safe-integer +//! range. Rust services that read bundle hashes //! must use the same grammar so malformed Redis state fails closed instead of //! silently normalizing to another worker version. //! -//! `routes_key` / `worker_versions_key` / `do_storage_id_key` mirror -//! `shared/version.js`'s `routesKey` / `workerVersionsKey` / -//! `doStorageIdKey`. Control owns these keys; Rust readers must build them -//! here so a future key-grammar change updates JS and Rust together. +//! `routes_key` / `worker_versions_key` / `worker_delete_lock_key` / +//! `do_storage_id_key` mirror `shared/version.js`'s lifecycle key builders. +//! Control owns these keys; Rust readers must build them here so a future +//! key-grammar change updates JS and Rust together. use std::fmt; +const MAX_SAFE_VERSION: u64 = 9_007_199_254_740_991; + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct InvalidVersionTag; @@ -28,7 +31,10 @@ pub fn parse_version_tag(version: &str) -> Result { if raw.is_empty() || raw.starts_with('0') || !raw.bytes().all(|byte| byte.is_ascii_digit()) { return Err(InvalidVersionTag); } - raw.parse::().map_err(|_| InvalidVersionTag) + let parsed = raw.parse::().map_err(|_| InvalidVersionTag)?; + (parsed <= MAX_SAFE_VERSION) + .then_some(parsed) + .ok_or(InvalidVersionTag) } pub fn worker_bundle_key( @@ -50,6 +56,11 @@ pub fn worker_versions_key(ns: &str, worker: &str) -> String { format!("worker-versions:{ns}:{worker}") } +/// Worker-scoped Control mutation lock shared with lifecycle readers. +pub fn worker_delete_lock_key(ns: &str, worker: &str) -> String { + format!("worker-delete-lock:{ns}:{worker}") +} + /// Logical Worker -> Durable Object storage pointer. Control owns writes; DO /// runtime and workflows read it for owner/storage fencing. pub fn do_storage_id_key(ns: &str, worker: &str) -> String { @@ -66,6 +77,21 @@ mod tests { assert_eq!(parse_version_tag("v42").unwrap(), 42); } + #[test] + fn matches_cross_language_version_fixture() { + let fixture: serde_json::Value = + serde_json::from_str(include_str!("../../../tests/fixtures/version-tags.json")) + .expect("version tag fixture parses"); + for case in fixture["cases"] + .as_array() + .expect("version tag fixture cases is an array") + { + let tag = case["tag"].as_str().expect("version tag is a string"); + let expected = case["parsed"].as_u64(); + assert_eq!(parse_version_tag(tag).ok(), expected, "{tag:?}"); + } + } + #[test] fn rejects_malformed_version_tags() { for version in ["", "v", "v0", "v01", "1", "V1", "v1a"] { @@ -92,6 +118,10 @@ mod tests { worker_versions_key("tenant", "worker"), "worker-versions:tenant:worker" ); + assert_eq!( + worker_delete_lock_key("tenant", "worker"), + "worker-delete-lock:tenant:worker" + ); assert_eq!( do_storage_id_key("tenant", "worker"), "worker:do-storage:tenant:worker" diff --git a/rust/redis-proxy/src/kv.rs b/rust/redis-proxy/src/kv.rs index 69b5d70..052df02 100644 --- a/rust/redis-proxy/src/kv.rs +++ b/rust/redis-proxy/src/kv.rs @@ -12,6 +12,7 @@ use serde_json::{Value, json}; use std::collections::HashMap; use wdl_rust_common::hash::fnv1a32; use wdl_rust_common::identity::is_valid_runtime_load_ns; +use wdl_rust_common::redis_eval::eval_cmd; use crate::observability::Metrics; use crate::{AppError, AppResult, AppState, SERVICE, empty}; @@ -166,12 +167,14 @@ fn meta_field(key: &str) -> String { format!("{META_FIELD_PREFIX}{key}") } +type GroupedHmgetCommand = (String, Vec, Vec); + fn grouped_hmget_commands( ns: &str, id: &str, keys: &[String], field_prefix: &str, -) -> Vec<(String, Vec, Vec)> { +) -> Vec { grouped_hmget_commands_for_indices(ns, id, keys, 0..keys.len(), field_prefix) } @@ -181,7 +184,7 @@ fn grouped_hmget_commands_for_indices( keys: &[String], indices: impl IntoIterator, field_prefix: &str, -) -> Vec<(String, Vec, Vec)> { +) -> Vec { let mut by_hash: HashMap, Vec)> = HashMap::new(); for index in indices { let key = &keys[index]; @@ -255,33 +258,137 @@ fn parse_hmget_budget_reply( Ok((bytes, out)) } -async fn hmget_with_raw_byte_budget( +fn hmget_with_raw_byte_budget_command( + redis_key: &str, + remaining: usize, + fields: &[String], +) -> redis::Cmd { + let remaining_arg = remaining.to_string(); + let mut args = Vec::with_capacity(fields.len() + 1); + args.push(remaining_arg.as_str()); + args.extend(fields.iter().map(String::as_str)); + eval_cmd(HMGET_WITH_RAW_BYTE_BUDGET_SCRIPT, &[redis_key], &args) +} + +async fn query_hmget_with_raw_byte_budget( conn: &mut ConnectionManager, - total: &mut usize, redis_key: &str, + remaining: usize, fields: &[String], -) -> AppResult>>> { +) -> AppResult<(usize, Vec>>)> { if fields.is_empty() { - return Ok(Vec::new()); + return Ok((0, Vec::new())); + } + // HSTRLEN and HMGET run in one Redis-side script, so an over-budget group + // is rejected before its payload bytes leave Redis. + let cmd = hmget_with_raw_byte_budget_command(redis_key, remaining, fields); + parse_hmget_budget_reply(cmd.query_async(conn).await?, fields.len()) +} + +#[derive(Default)] +struct RawByteBudget { + preflight_bytes: usize, + actual_bytes: usize, +} + +impl RawByteBudget { + fn remaining(&self) -> AppResult { + KV_BATCH_RAW_BYTES_MAX + .checked_sub(self.preflight_bytes) + .ok_or_else(raw_bytes_too_large) + } + + fn record_preflight(&mut self, bytes: usize) -> AppResult<()> { + add_batch_raw_bytes(&mut self.preflight_bytes, bytes) + } + + fn record_actual(&mut self, value: Option<&[u8]>) -> AppResult<()> { + if let Some(bytes) = value { + add_batch_raw_bytes(&mut self.actual_bytes, bytes.len())?; + } + Ok(()) + } +} + +struct GroupedHmgetRequest { + redis_key: String, + indices: Vec, + fields: Vec, + remaining: usize, +} + +struct GroupedFieldRead { + pending: std::vec::IntoIter, + output: Vec>>, +} + +impl GroupedFieldRead { + fn new(commands: Vec, output_len: usize) -> Self { + Self { + pending: commands.into_iter(), + output: vec![None; output_len], + } + } + + fn next_request(&mut self, budget: &RawByteBudget) -> AppResult> { + let Some((redis_key, indices, fields)) = self.pending.next() else { + return Ok(None); + }; + if indices.len() != fields.len() || indices.iter().any(|index| *index >= self.output.len()) + { + return Err(AppError::internal_error("invalid grouped KV HMGET plan")); + } + Ok(Some(GroupedHmgetRequest { + redis_key, + indices, + fields, + remaining: budget.remaining()?, + })) } - // The Lua preflight is the Redis-side atomic budget gate: HSTRLEN and HMGET - // run in one script, so over-budget batches fail before payload bytes leave - // Redis. Callers keep a separate actual-byte counter over returned values as - // defense-in-depth without double-counting stable data against the budget. - let remaining = KV_BATCH_RAW_BYTES_MAX - .checked_sub(*total) - .ok_or_else(raw_bytes_too_large)?; - let mut cmd = redis::cmd("EVAL"); - cmd.arg(HMGET_WITH_RAW_BYTE_BUDGET_SCRIPT) - .arg(1) - .arg(redis_key) - .arg(remaining); - for field in fields { - cmd.arg(field); + + fn apply_response( + &mut self, + budget: &mut RawByteBudget, + request: GroupedHmgetRequest, + preflight_bytes: usize, + values: Vec>>, + ) -> AppResult<()> { + if values.len() != request.indices.len() { + return Err(AppError::internal_error("invalid grouped KV HMGET reply")); + } + budget.record_preflight(preflight_bytes)?; + for (index, value) in request.indices.into_iter().zip(values) { + budget.record_actual(value.as_deref())?; + self.output[index] = value; + } + Ok(()) + } + + fn finish(self) -> Vec>> { + self.output + } +} + +async fn load_grouped_fields_with_raw_byte_budget( + conn: &mut ConnectionManager, + commands: Vec, + output_len: usize, + budget: &mut RawByteBudget, +) -> AppResult>>> { + // The preflight total limits transfer between groups; the separate actual + // total rechecks returned payloads before callers encode a response. + let mut read = GroupedFieldRead::new(commands, output_len); + while let Some(request) = read.next_request(budget)? { + let (preflight_bytes, values) = query_hmget_with_raw_byte_budget( + conn, + &request.redis_key, + request.remaining, + &request.fields, + ) + .await?; + read.apply_response(budget, request, preflight_bytes, values)?; } - let (bytes, values) = parse_hmget_budget_reply(cmd.query_async(conn).await?, fields.len())?; - add_batch_raw_bytes(total, bytes)?; - Ok(values) + Ok(read.finish()) } pub(crate) fn kv_put_pipeline( @@ -360,6 +467,12 @@ fn decode_metadata(bytes: Option>) -> AppResult> { .transpose() } +fn normalize_list_prefix(prefix: Option) -> AppResult { + let prefix = prefix.unwrap_or_default(); + validate_kv_key(&prefix)?; + Ok(prefix) +} + pub(crate) fn escape_glob_literal(input: &str) -> String { let mut out = String::with_capacity(input.len()); for ch in input.chars() { @@ -396,6 +509,7 @@ pub(crate) async fn kv_get( ); response.headers_mut().insert( CONTENT_LENGTH, + // `usize::to_string()` is decimal ASCII, which is always a valid header value. HeaderValue::from_str(&len.to_string()).unwrap(), ); Ok(response) @@ -452,55 +566,48 @@ pub(crate) async fn kv_get_batch( let include_metadata = body.metadata.unwrap_or(false); let keys = body.keys; let mut conn = state.redis(); - let mut preflight_raw_bytes = 0_usize; - let mut actual_raw_bytes = 0_usize; - let mut values: Vec>> = vec![None; keys.len()]; - for (redis_key, indices, fields) in - grouped_hmget_commands(&q.ns, &q.id, &keys, VALUE_FIELD_PREFIX) - { - let batch = - hmget_with_raw_byte_budget(&mut conn, &mut preflight_raw_bytes, &redis_key, &fields) - .await?; - for (offset, value) in batch.into_iter().enumerate() { - if let Some(bytes) = value.as_ref() { - add_batch_raw_bytes(&mut actual_raw_bytes, bytes.len())?; - } - values[indices[offset]] = value; - } - } + let mut budget = RawByteBudget::default(); + let value_commands = grouped_hmget_commands(&q.ns, &q.id, &keys, VALUE_FIELD_PREFIX); + let mut values = load_grouped_fields_with_raw_byte_budget( + &mut conn, + value_commands, + keys.len(), + &mut budget, + ) + .await?; let mut metadata: Vec> = vec![None; keys.len()]; if include_metadata { let present_value_indices = values .iter() .enumerate() - .filter_map(|(index, value)| value.as_ref().map(|_| index)); - for (redis_key, indices, fields) in grouped_hmget_commands_for_indices( + .filter_map(|(index, value)| value.as_ref().map(|_| index)) + .collect::>(); + let metadata_commands = grouped_hmget_commands_for_indices( &q.ns, &q.id, &keys, present_value_indices, META_FIELD_PREFIX, - ) { - let batch = hmget_with_raw_byte_budget( - &mut conn, - &mut preflight_raw_bytes, - &redis_key, - &fields, - ) - .await?; - for (offset, raw) in batch.into_iter().enumerate() { - let index = indices[offset]; - if values[index].is_none() { - continue; - } - if let Some(bytes) = raw.as_ref() { - add_batch_raw_bytes(&mut actual_raw_bytes, bytes.len())?; - } + ); + let raw_metadata = load_grouped_fields_with_raw_byte_budget( + &mut conn, + metadata_commands, + keys.len(), + &mut budget, + ) + .await?; + for (index, raw) in raw_metadata.into_iter().enumerate() { + if values[index].is_some() { metadata[index] = decode_metadata(raw)?; } } } - record_kv_value_bytes(state.metrics(), "get_batch", "raw_batch", actual_raw_bytes); + record_kv_value_bytes( + state.metrics(), + "get_batch", + "raw_batch", + budget.actual_bytes, + ); let mut entries = Vec::with_capacity(keys.len()); for (index, key) in keys.into_iter().enumerate() { let value = values[index].take(); @@ -586,8 +693,7 @@ pub(crate) async fn kv_list( q.validate_scope()?; let limit = normalize_list_limit(q.limit) as usize; let (mut bucket, mut scan_cursor, mut overflow) = decode_list_cursor(q.cursor)?; - let prefix = q.prefix.unwrap_or_default(); - validate_kv_key(&prefix)?; + let prefix = normalize_list_prefix(q.prefix)?; let include_metadata = q.metadata.unwrap_or(false); let pattern = format!("{VALUE_FIELD_PREFIX}{}*", escape_glob_literal(&prefix)); let mut raw_fields = Vec::new(); @@ -648,27 +754,25 @@ pub(crate) async fn kv_list( .collect::>>()?; let mut metadata = if include_metadata { let mut conn = state.redis(); - let mut preflight_raw_bytes = 0_usize; - let mut actual_raw_bytes = 0_usize; - let mut values: Vec> = vec![None; user_keys.len()]; - for (redis_key, indices, fields) in - grouped_hmget_commands(&q.ns, &q.id, &user_keys, META_FIELD_PREFIX) - { - let batch = hmget_with_raw_byte_budget( - &mut conn, - &mut preflight_raw_bytes, - &redis_key, - &fields, - ) - .await?; - for (offset, raw) in batch.into_iter().enumerate() { - if let Some(bytes) = raw.as_ref() { - add_batch_raw_bytes(&mut actual_raw_bytes, bytes.len())?; - } - values[indices[offset]] = decode_metadata(raw)?; - } - } - record_kv_value_bytes(state.metrics(), "list", "metadata_batch", actual_raw_bytes); + let mut budget = RawByteBudget::default(); + let metadata_commands = grouped_hmget_commands(&q.ns, &q.id, &user_keys, META_FIELD_PREFIX); + let raw_metadata = load_grouped_fields_with_raw_byte_budget( + &mut conn, + metadata_commands, + user_keys.len(), + &mut budget, + ) + .await?; + let values = raw_metadata + .into_iter() + .map(decode_metadata) + .collect::>>()?; + record_kv_value_bytes( + state.metrics(), + "list", + "metadata_batch", + budget.actual_bytes, + ); values } else { Vec::new() @@ -778,57 +882,116 @@ mod tests { } #[test] - fn kv_batch_get_checks_budget_between_value_reads() { - let source = include_str!("kv.rs"); - let start = source - .find("pub(crate) async fn kv_get_batch") - .expect("kv_get_batch route must exist"); - let end = source[start..] - .find("pub(crate) async fn kv_put") - .expect("kv_put should follow kv_get_batch") - + start; - let helper = &source[start..end]; - assert!( - helper.matches("hmget_with_raw_byte_budget").count() >= 2, - "kv_get_batch must atomically budget-check value and metadata HMGET payloads" - ); - assert!( - helper.matches("add_batch_raw_bytes").count() >= 2, - "kv_get_batch must also re-check actual HMGET payload bytes after reading" - ); - assert!( - !helper.contains("redis::pipe"), - "kv_get_batch must not pipeline every bucket before checking the aggregate byte budget" + fn kv_binding_id_grammar_matches_cross_language_fixture() { + let fixture: Value = serde_json::from_str(include_str!( + "../../../tests/fixtures/cross-language-identity.json" + )) + .expect("cross-language identity fixture must parse"); + let cases = fixture["kvIds"] + .as_array() + .expect("kvIds fixture field must be an array"); + for case in cases { + let value = case["value"] + .as_str() + .expect("kvIds fixture value must be a string"); + let valid = case["valid"] + .as_bool() + .expect("kvIds fixture valid flag must be a boolean"); + assert_eq!(is_valid_kv_binding_id(value), valid, "kvIds:{value:?}"); + } + } + + #[test] + fn hmget_budget_command_packs_script_key_budget_and_fields() { + let fields = vec!["v:first".to_string(), "v:second".to_string()]; + let commands = parse_packed_commands( + &hmget_with_raw_byte_budget_command("kvh:tenant:store:b:1", 1024, &fields) + .get_packed_command(), ); - assert!( - source.contains("redis.call(\"HMGET\""), - "the atomic KV budget helper should batch reads with Redis HMGET" + + assert_eq!(commands.len(), 1); + let command = &commands[0]; + assert_eq!(command[0], "EVAL"); + assert_eq!(command[1], HMGET_WITH_RAW_BYTE_BUDGET_SCRIPT); + assert_eq!( + &command[2..], + ["1", "kvh:tenant:store:b:1", "1024", "v:first", "v:second"] ); } #[test] - fn kv_list_metadata_batches_metadata_reads() { - let source = include_str!("kv.rs"); - let start = source - .find("pub(crate) async fn kv_list") - .expect("kv_list route must exist"); - let end = source[start..] - .find("#[cfg(test)]") - .expect("tests should follow kv_list") - + start; - let helper = &source[start..end]; - assert!( - helper.contains("grouped_hmget_commands"), - "kv_list metadata reads should group metadata fields by hash bucket" - ); - assert!( - helper.contains("hmget_with_raw_byte_budget"), - "kv_list metadata reads must atomically budget-check raw metadata bytes before returning payloads" + fn grouped_hmget_commands_batches_metadata_fields_by_hash_bucket() { + let keys = vec![ + "alpha".to_string(), + "bravo".to_string(), + "charlie".to_string(), + ]; + let commands = grouped_hmget_commands("tenant", "store", &keys, META_FIELD_PREFIX); + assert_eq!( + commands + .iter() + .map(|(_, indices, fields)| { + assert_eq!(indices.len(), fields.len()); + fields.len() + }) + .sum::(), + keys.len() ); - assert!( - helper.contains("add_batch_raw_bytes"), - "kv_list metadata reads must also re-check actual HMGET payload bytes" + + for (redis_key, indices, fields) in commands { + for (offset, index) in indices.iter().enumerate() { + assert_eq!( + redis_key, + hash_key_for_user_key("tenant", "store", &keys[*index]) + ); + assert_eq!(fields[offset], meta_field(&keys[*index])); + } + } + } + + #[test] + fn grouped_budgeted_reads_decrement_budget_and_restore_input_order() { + let commands = vec![ + ( + "hash-a".to_string(), + vec![2, 0], + vec!["v:charlie".to_string(), "v:alpha".to_string()], + ), + ("hash-b".to_string(), vec![1], vec!["v:bravo".to_string()]), + ]; + let mut read = GroupedFieldRead::new(commands, 3); + let mut budget = RawByteBudget::default(); + + let first = read.next_request(&budget).unwrap().unwrap(); + assert_eq!(first.redis_key, "hash-a"); + assert_eq!(first.remaining, KV_BATCH_RAW_BYTES_MAX); + assert_eq!(first.fields, ["v:charlie", "v:alpha"]); + read.apply_response( + &mut budget, + first, + 1, + vec![Some(b"ccc".to_vec()), Some(b"aa".to_vec())], + ) + .unwrap(); + + let second = read.next_request(&budget).unwrap().unwrap(); + assert_eq!(second.redis_key, "hash-b"); + assert_eq!(second.remaining, KV_BATCH_RAW_BYTES_MAX - 1); + assert_eq!(second.fields, ["v:bravo"]); + read.apply_response(&mut budget, second, 2, vec![Some(b"bbbb".to_vec())]) + .unwrap(); + + assert!(read.next_request(&budget).unwrap().is_none()); + assert_eq!( + read.finish(), + vec![ + Some(b"aa".to_vec()), + Some(b"bbbb".to_vec()), + Some(b"ccc".to_vec()), + ] ); + assert_eq!(budget.preflight_bytes, 3); + assert_eq!(budget.actual_bytes, 9); } #[test] @@ -889,33 +1052,14 @@ mod tests { #[test] fn kv_list_prefix_uses_kv_key_byte_limit() { - let source = include_str!("kv.rs"); - let start = source - .find("pub(crate) async fn kv_list") - .expect("kv_list route must exist"); - let end = source[start..] - .find("let include_metadata") - .expect("kv_list prefix validation should happen before HSCAN setup") - + start; - assert!( - source[start..end].contains("validate_kv_key(&prefix)?"), - "kv_list must reject oversized prefixes before composing the HSCAN pattern" + assert_eq!(normalize_list_prefix(None).unwrap(), ""); + assert_eq!( + normalize_list_prefix(Some("photos/".to_string())).unwrap(), + "photos/" ); let oversized = "x".repeat(KV_KEY_MAX_BYTES + 1); - let q = KvParams { - ns: "tenant".to_string(), - id: "store".to_string(), - key: None, - ttl: None, - exat: None, - prefix: Some(oversized), - limit: None, - metadata: None, - cursor: None, - }; - let prefix = q.prefix.unwrap_or_default(); - let err = validate_kv_key(&prefix).unwrap_err(); + let err = normalize_list_prefix(Some(oversized)).unwrap_err(); assert_eq!(err.status, StatusCode::BAD_REQUEST); assert!(err.message.contains("KV key exceeds 512 byte limit")); } @@ -972,52 +1116,6 @@ mod tests { assert_eq!(err.code, "invalid_request"); } - #[test] - fn kv_list_cursor_overflow_existence_probe_does_not_fetch_values() { - let source = include_str!("kv/cursor.rs"); - let start = source - .find("async fn existing_cursor_overflow_fields") - .expect("existing_cursor_overflow_fields helper must exist"); - let helper = &source[start..]; - assert!( - !helper.contains("candidates.clone()"), - "cursor overflow validation should not clone the overflow candidate vector" - ); - assert!( - !helper.contains("\"MGET\""), - "cursor overflow validation must not fetch KV values" - ); - assert!( - helper.contains("\"HEXISTS\""), - "cursor overflow validation should only probe field existence" - ); - } - - #[test] - fn kv_list_metadata_checks_budget_between_metadata_reads() { - let source = include_str!("kv.rs"); - let start = source - .find("pub(crate) async fn kv_list") - .expect("kv_list route must exist"); - let end = source[start..] - .find("#[cfg(test)]") - .expect("test module should follow kv_list") - + start; - let helper = &source[start..end]; - assert!( - helper.contains("hmget_with_raw_byte_budget"), - "kv_list metadata must enforce the aggregate raw metadata byte budget inside the HMGET helper" - ); - assert!( - helper.contains("add_batch_raw_bytes"), - "kv_list metadata must re-check actual HMGET payload bytes after reading" - ); - assert!( - !helper.contains("redis::pipe"), - "kv_list metadata must not pipeline all metadata before checking the aggregate byte budget" - ); - } - #[test] fn kv_put_pipeline_preserves_value_and_metadata_ttl_contract() { let commands = parse_packed_commands( diff --git a/rust/redis-proxy/src/kv/cursor.rs b/rust/redis-proxy/src/kv/cursor.rs index df19e95..a89c471 100644 --- a/rust/redis-proxy/src/kv/cursor.rs +++ b/rust/redis-proxy/src/kv/cursor.rs @@ -75,14 +75,9 @@ pub(crate) async fn existing_cursor_overflow_fields( if candidates.is_empty() { return Ok(Vec::new()); } + let pipe = existing_cursor_overflow_fields_pipeline(&hash_key, &candidates); let exists: Vec = state - .with_redis(async |mut conn| { - let mut pipe = redis::pipe(); - for field in &candidates { - pipe.cmd("HEXISTS").arg(&hash_key).arg(field); - } - pipe.query_async(&mut conn).await - }) + .with_redis(async |mut conn| pipe.query_async(&mut conn).await) .await?; Ok(candidates .into_iter() @@ -90,3 +85,37 @@ pub(crate) async fn existing_cursor_overflow_fields( .filter_map(|(key, exists)| (exists > 0).then_some(key)) .collect()) } + +fn existing_cursor_overflow_fields_pipeline( + hash_key: &str, + candidates: &[String], +) -> redis::Pipeline { + let mut pipe = redis::pipe(); + for field in candidates { + pipe.cmd("HEXISTS").arg(hash_key).arg(field); + } + pipe +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::parse_packed_commands; + + #[test] + fn existing_cursor_overflow_fields_pipeline_only_probes_existence() { + let candidates = vec!["v:first".to_string(), "v:second".to_string()]; + let commands = parse_packed_commands( + &existing_cursor_overflow_fields_pipeline("kvh:tenant:store:b:1", &candidates) + .get_packed_pipeline(), + ); + + assert_eq!( + commands, + [ + ["HEXISTS", "kvh:tenant:store:b:1", "v:first"], + ["HEXISTS", "kvh:tenant:store:b:1", "v:second"], + ] + ); + } +} diff --git a/rust/redis-proxy/src/lib.rs b/rust/redis-proxy/src/lib.rs index 56b56fa..b66e38e 100644 --- a/rust/redis-proxy/src/lib.rs +++ b/rust/redis-proxy/src/lib.rs @@ -7,7 +7,7 @@ use axum::body::Body; use axum::extract::DefaultBodyLimit; use axum::extract::State; use axum::http::header::CONTENT_LENGTH; -use axum::http::{HeaderMap, HeaderValue, Request, StatusCode}; +use axum::http::{HeaderValue, Request, StatusCode}; use axum::middleware::{self, Next}; use axum::response::{IntoResponse, Response}; use axum::routing::{delete, get, post, put}; @@ -16,11 +16,12 @@ use serde_json::{Value, json}; use wdl_rust_common::env::env_u16; use wdl_rust_common::health::healthcheck_http_200; use wdl_rust_common::internal_auth::{ - InternalAuthTokens, internal_auth_headers_match, internal_auth_tokens_from_env, + INTERNAL_AUTH_FAILURE_CODE, INTERNAL_AUTH_FAILURE_MESSAGE, InternalAuthTokens, + internal_auth_failure_response, internal_auth_headers_match, internal_auth_tokens_from_env, }; use wdl_rust_common::metrics::prometheus_response; use wdl_rust_common::redis_conn::RedisConnection; -use wdl_rust_common::request_id::sanitize_request_id; +use wdl_rust_common::request_id::request_id_from_headers; use wdl_rust_common::time::duration_ms_for_log; // CF KV allows 25 MiB values; set Axum's body cap above that instead of @@ -292,13 +293,6 @@ fn response_error(response: &Response) -> Option<(&'static str, &str)> { .map(|err| (err.code, err.message.as_str())) } -fn request_id_from_headers(headers: &HeaderMap) -> Option { - headers - .get("x-request-id") - .and_then(|value| value.to_str().ok()) - .and_then(sanitize_request_id) -} - async fn track_request( State(state): State, request: Request, @@ -311,14 +305,7 @@ async fn track_request( if !matches!(route, "healthz" | "metrics") && !internal_auth_headers_match(request.headers(), &state.internal_auth_tokens) { - let response = ( - StatusCode::UNAUTHORIZED, - Json(json!({ - "error": "internal_auth_failed", - "message": "Internal authentication failed", - })), - ) - .into_response(); + let response = internal_auth_failure_response(); record_request_complete( &state, method.as_str(), @@ -326,7 +313,7 @@ async fn track_request( response.status(), request_id.as_deref(), started_at, - Some(("internal_auth_failed", "Internal authentication failed")), + Some((INTERNAL_AUTH_FAILURE_CODE, INTERNAL_AUTH_FAILURE_MESSAGE)), ); return response; } @@ -417,6 +404,7 @@ pub async fn run() -> Result<(), Box> { #[cfg(test)] mod tests { use axum::body::to_bytes; + use axum::http::HeaderMap; use axum::response::IntoResponse; use serde_json::json; diff --git a/rust/redis-proxy/src/queue.rs b/rust/redis-proxy/src/queue.rs index 7a68d58..b87448b 100644 --- a/rust/redis-proxy/src/queue.rs +++ b/rust/redis-proxy/src/queue.rs @@ -6,8 +6,9 @@ use redis::Pipeline; use serde::{Deserialize, Serialize}; use wdl_rust_common::identity::is_valid_runtime_load_ns; use wdl_rust_common::queue_keys::{ - QUEUE_DELAYED_INDEX_KEY, QUEUE_STREAM_INDEX_KEY, is_valid_queue_name, queue_delayed_key, - queue_stream_key, + QUEUE_DELAYED_INDEX_KEY, QUEUE_DELAYED_WAKE_KEY_FIELD, QUEUE_DELAYED_WAKE_STREAM, + QUEUE_DELAYED_WAKE_VISIBLE_AT_FIELD, QUEUE_STREAM_INDEX_KEY, is_valid_queue_name, + queue_delayed_key, queue_stream_key, }; use crate::{AppError, AppResult, AppState, empty}; @@ -15,7 +16,6 @@ use crate::{AppError, AppResult, AppState, empty}; pub(crate) const MAX_QUEUE_MESSAGE_BYTES: usize = 128_000; pub(crate) const MAX_QUEUE_BATCH_MESSAGES: usize = 100; pub(crate) const MAX_QUEUE_BATCH_BYTES: usize = 256_000; -pub(crate) const QUEUE_DELAYED_WAKE_STREAM: &str = "queue-delayed-wake"; const QUEUE_DELAYED_WAKE_MAX_LEN: usize = 1000; const MAX_SAFE_VISIBLE_AT: f64 = 9_007_199_254_740_991.0; @@ -63,9 +63,9 @@ pub(crate) fn pipe_delayed_wake(pipe: &mut Pipeline, delayed_key: &str, visible_ .arg("~") .arg(QUEUE_DELAYED_WAKE_MAX_LEN) .arg("*") - .arg("delayed_key") + .arg(QUEUE_DELAYED_WAKE_KEY_FIELD) .arg(delayed_key) - .arg("visible_at") + .arg(QUEUE_DELAYED_WAKE_VISIBLE_AT_FIELD) .arg(visible_at.to_string()); } @@ -399,9 +399,9 @@ mod tests { let packed = String::from_utf8(pipe.get_packed_pipeline()).unwrap(); assert!(packed.contains(QUEUE_DELAYED_WAKE_STREAM)); assert!(packed.contains("MAXLEN")); - assert!(packed.contains("delayed_key")); + assert!(packed.contains(QUEUE_DELAYED_WAKE_KEY_FIELD)); assert!(packed.contains("queue-delayed:demo:jobs")); - assert!(packed.contains("visible_at")); + assert!(packed.contains(QUEUE_DELAYED_WAKE_VISIBLE_AT_FIELD)); } #[test] diff --git a/rust/redis-proxy/src/secrets.rs b/rust/redis-proxy/src/secrets.rs index 056e344..d6b16fd 100644 --- a/rust/redis-proxy/src/secrets.rs +++ b/rust/redis-proxy/src/secrets.rs @@ -6,7 +6,7 @@ use aes_gcm::aead::AeadInOut; use aes_gcm::{Aes256Gcm, KeyInit, Nonce, Tag}; use base64::Engine; use base64::engine::general_purpose::STANDARD as BASE64; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use wdl_rust_common::hash::fnv1a64; use zeroize::Zeroizing; @@ -28,7 +28,7 @@ const DEK_CACHE_SHARDS: usize = 16; const DEK_CACHE_SHARD_LIMIT: usize = DEK_CACHE_LIMIT / DEK_CACHE_SHARDS; const DEK_CACHE_TTL: Duration = Duration::from_secs(60); -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, Serialize)] #[serde(deny_unknown_fields)] struct SecretEnvelope { v: u8, @@ -123,6 +123,13 @@ impl SecretEnvelopeDecryptor { let json_bytes = &value[SECRET_ENVELOPE_PREFIX.len()..]; let envelope: SecretEnvelope = serde_json::from_slice(json_bytes) .map_err(|_| secret_decrypt_error("secret envelope JSON is invalid"))?; + let canonical_json = serde_json::to_vec(&envelope) + .map_err(|_| secret_decrypt_error("secret envelope JSON is invalid"))?; + if canonical_json != json_bytes { + return Err(secret_decrypt_error( + "secret envelope JSON is not canonical", + )); + } validate_envelope(&envelope)?; let provider = self.local.as_ref().ok_or_else(|| { secret_decrypt_error("secret envelope local provider is not configured") @@ -321,6 +328,8 @@ fn aes_gcm_decrypt(key: &[u8], iv: &[u8], ct: &[u8], tag: &[u8], aad: &[u8]) -> .map_err(|_| secret_decrypt_error("secret envelope has invalid AES-GCM material"))?; let tag = Tag::try_from(tag) .map_err(|_| secret_decrypt_error("secret envelope has invalid AES-GCM material"))?; + // Decrypt into the ciphertext clone and pass the tag separately so large + // secrets do not require a second ct||tag message allocation. cipher .decrypt_inout_detached(&nonce, aad, plaintext.as_mut_slice().into(), &tag) .map_err(|_| secret_decrypt_error("secret envelope authentication failed"))?; @@ -350,6 +359,59 @@ fn secret_config_error(message: impl Into) -> AppError { mod tests { use super::*; + #[derive(Deserialize)] + #[serde(rename_all = "camelCase")] + struct SecretEnvelopeParityFixture { + provider: SecretEnvelopeParityProvider, + vectors: Vec, + rejections: Vec, + } + + #[derive(Deserialize)] + #[serde(rename_all = "camelCase")] + struct SecretEnvelopeParityProvider { + kid: String, + local_key_b64: String, + } + + #[derive(Deserialize)] + #[serde(rename_all = "camelCase")] + struct SecretEnvelopeParityVector { + name: String, + plaintext: String, + hash_key: String, + field_name: String, + envelope: String, + } + + #[derive(Deserialize)] + #[serde(rename_all = "camelCase")] + struct SecretEnvelopeParityRejection { + name: String, + envelope: String, + hash_key: String, + field_name: String, + configured_kid: String, + } + + fn parity_fixture() -> SecretEnvelopeParityFixture { + serde_json::from_str(include_str!( + "../../../tests/fixtures/secret-envelope-parity.json" + )) + .expect("secret envelope parity fixture must parse") + } + + fn parity_vector<'a>( + fixture: &'a SecretEnvelopeParityFixture, + name: &str, + ) -> &'a SecretEnvelopeParityVector { + fixture + .vectors + .iter() + .find(|vector| vector.name == name) + .expect("secret envelope parity vector must exist") + } + fn test_metrics() -> Arc { Arc::new(Metrics::default()) } @@ -362,11 +424,23 @@ mod tests { } } - fn decryptor_with_local(kid: &str) -> SecretEnvelopeDecryptor { + fn decryptor_with_provider( + provider: &SecretEnvelopeParityProvider, + kid: &str, + ) -> SecretEnvelopeDecryptor { + let decoded = decode_canonical_base64( + &provider.local_key_b64, + SECRET_ENVELOPE_LOCAL_KEY_ENV, + false, + ) + .expect("secret envelope parity provider key must be canonical base64"); + let key: [u8; AES_256_KEY_BYTES] = decoded + .try_into() + .expect("secret envelope parity provider key must be 32 bytes"); SecretEnvelopeDecryptor { local: Some(LocalProvider { kid: kid.to_string(), - key: Zeroizing::new(*b"0123456789abcdef0123456789abcdef"), + key: Zeroizing::new(key), }), dek_cache: Arc::new(dek_cache_shards()), metrics: test_metrics(), @@ -413,89 +487,62 @@ mod tests { } #[test] - fn decrypts_js_generated_envelope_vector() { - let decryptor = decryptor_with_local("local:test:secret-envelope:v1"); - let envelope = concat!( - r#"WDL-ENC:{"v":1,"alg":"AES-256-GCM","kid":"local:test:secret-envelope:v1","#, - r#""edek":"ICEiIyQlJicoKSorrmiFFNwSNL789zDTZysVGjsTpksuXqulg0Mt7JrBiLTG0wE1moqP31Jd+edymdiU","#, - r#""iv":"LC0uLzAxMjM0NTY3","ct":"IZhLZQvljbpT5euHV5YP","tag":"0wuyjfzeZfuY5hD3HRoKdg=="}"# - ); - let out = decryptor - .decrypt_hash_entries( - "secrets:demo:api", - vec![("TOKEN".to_string(), envelope.as_bytes().to_vec())], - ) - .unwrap(); - assert_eq!(out.get("TOKEN").unwrap(), "sensitive-value"); - } - - #[test] - fn decrypts_js_generated_empty_string_vector() { - let decryptor = decryptor_with_local("local:test:secret-envelope:v1"); - let envelope = concat!( - r#"WDL-ENC:{"v":1,"alg":"AES-256-GCM","kid":"local:test:secret-envelope:v1","#, - r#""edek":"ICEiIyQlJicoKSorrmiFFNwSNL789zDTZysVGjsTpksuXqulg0Mt7JrBiLSmHuPVe4uS7JepH053l0Yt","#, - r#""iv":"LC0uLzAxMjM0NTY3","ct":"","tag":"TTVUTxM6DfSV+VZ/AODK1w=="}"# - ); - let out = decryptor - .decrypt_hash_entries( - "secrets:demo:api", - vec![("EMPTY".to_string(), envelope.as_bytes().to_vec())], - ) - .unwrap(); - assert_eq!(out.get("EMPTY").unwrap(), ""); - } - - #[test] - fn decrypt_rejects_storage_location_mismatch() { - let decryptor = decryptor_with_local("local:test:secret-envelope:v1"); - let envelope = concat!( - r#"WDL-ENC:{"v":1,"alg":"AES-256-GCM","kid":"local:test:secret-envelope:v1","#, - r#""edek":"ICEiIyQlJicoKSorrmiFFNwSNL789zDTZysVGjsTpksuXqulg0Mt7JrBiLTG0wE1moqP31Jd+edymdiU","#, - r#""iv":"LC0uLzAxMjM0NTY3","ct":"IZhLZQvljbpT5euHV5YP","tag":"0wuyjfzeZfuY5hD3HRoKdg=="}"# - ); - let err = decryptor - .decrypt_hash_entries( - "secrets:other:api", - vec![("TOKEN".to_string(), envelope.as_bytes().to_vec())], - ) - .unwrap_err(); - assert_eq!(err.code, "secret_decrypt_failed"); + fn decrypts_secret_envelope_parity_vectors() { + let fixture = parity_fixture(); + for vector in &fixture.vectors { + let decryptor = decryptor_with_provider(&fixture.provider, &fixture.provider.kid); + let out = decryptor + .decrypt_hash_entries( + &vector.hash_key, + vec![( + vector.field_name.clone(), + vector.envelope.as_bytes().to_vec(), + )], + ) + .unwrap_or_else(|err| panic!("{}: {err:?}", vector.name)); + assert_eq!( + out.get(&vector.field_name), + Some(&vector.plaintext), + "{}", + vector.name + ); + } } #[test] - fn decrypt_rejects_unknown_kid() { - let decryptor = decryptor_with_local("local:test:secret-envelope:v2"); - let envelope = concat!( - r#"WDL-ENC:{"v":1,"alg":"AES-256-GCM","kid":"local:test:secret-envelope:v1","#, - r#""edek":"ICEiIyQlJicoKSorrmiFFNwSNL789zDTZysVGjsTpksuXqulg0Mt7JrBiLTG0wE1moqP31Jd+edymdiU","#, - r#""iv":"LC0uLzAxMjM0NTY3","ct":"IZhLZQvljbpT5euHV5YP","tag":"0wuyjfzeZfuY5hD3HRoKdg=="}"# - ); - let err = decryptor - .decrypt_hash_entries( - "secrets:demo:api", - vec![("TOKEN".to_string(), envelope.as_bytes().to_vec())], - ) - .unwrap_err(); - assert_eq!(err.code, "secret_decrypt_failed"); + fn rejects_secret_envelope_parity_vectors() { + let fixture = parity_fixture(); + for rejection in &fixture.rejections { + let decryptor = decryptor_with_provider(&fixture.provider, &rejection.configured_kid); + let err = decryptor + .decrypt_hash_entries( + &rejection.hash_key, + vec![( + rejection.field_name.clone(), + rejection.envelope.as_bytes().to_vec(), + )], + ) + .unwrap_err(); + assert_eq!(err.code, "secret_decrypt_failed", "{}", rejection.name); + } } #[test] fn dek_cache_is_scoped_to_storage_location() { - let decryptor = decryptor_with_local("local:test:secret-envelope:v1"); - let envelope = concat!( - r#"WDL-ENC:{"v":1,"alg":"AES-256-GCM","kid":"local:test:secret-envelope:v1","#, - r#""edek":"ICEiIyQlJicoKSorrmiFFNwSNL789zDTZysVGjsTpksuXqulg0Mt7JrBiLTG0wE1moqP31Jd+edymdiU","#, - r#""iv":"LC0uLzAxMjM0NTY3","ct":"IZhLZQvljbpT5euHV5YP","tag":"0wuyjfzeZfuY5hD3HRoKdg=="}"# - ); + let fixture = parity_fixture(); + let vector = parity_vector(&fixture, "value"); + let decryptor = decryptor_with_provider(&fixture.provider, &fixture.provider.kid); // First read at the matching location decrypts and caches the DEK. let out = decryptor .decrypt_hash_entries( - "secrets:demo:api", - vec![("TOKEN".to_string(), envelope.as_bytes().to_vec())], + &vector.hash_key, + vec![( + vector.field_name.clone(), + vector.envelope.as_bytes().to_vec(), + )], ) .unwrap(); - assert_eq!(out.get("TOKEN").unwrap(), "sensitive-value"); + assert_eq!(out.get(&vector.field_name), Some(&vector.plaintext)); assert_eq!( decryptor .dek_cache @@ -508,18 +555,24 @@ mod tests { // Same envelope at the same location hits the cache. let again = decryptor .decrypt_hash_entries( - "secrets:demo:api", - vec![("TOKEN".to_string(), envelope.as_bytes().to_vec())], + &vector.hash_key, + vec![( + vector.field_name.clone(), + vector.envelope.as_bytes().to_vec(), + )], ) .unwrap(); - assert_eq!(again.get("TOKEN").unwrap(), "sensitive-value"); + assert_eq!(again.get(&vector.field_name), Some(&vector.plaintext)); // Same edek at a different location is a cache miss (the key is scoped // to hash_key/field) and fails closed at DEK decryption via data_key_aad. let err = decryptor .decrypt_hash_entries( "secrets:other:api", - vec![("TOKEN".to_string(), envelope.as_bytes().to_vec())], + vec![( + vector.field_name.clone(), + vector.envelope.as_bytes().to_vec(), + )], ) .unwrap_err(); assert_eq!(err.code, "secret_decrypt_failed"); @@ -533,26 +586,11 @@ mod tests { )); } - #[test] - fn aes_gcm_decrypt_uses_detached_tag_without_concat_message() { - let source = include_str!("secrets.rs"); - let helper = source - .split("fn aes_gcm_decrypt") - .nth(1) - .expect("aes_gcm_decrypt helper should exist"); - assert!( - helper.contains("decrypt_inout_detached"), - "AES-GCM decrypt should pass the tag separately instead of concatenating ct||tag" - ); - assert!( - !helper.contains("extend_from_slice(tag)"), - "AES-GCM decrypt should not allocate a ct||tag message buffer" - ); - } - #[test] fn decrypt_records_expired_dek_cache_evictions() { - let decryptor = decryptor_with_local("local:test:secret-envelope:v1"); + let fixture = parity_fixture(); + let vector = parity_vector(&fixture, "value"); + let decryptor = decryptor_with_provider(&fixture.provider, &fixture.provider.kid); { let mut cache = decryptor.dek_cache[0] .lock() @@ -567,18 +605,16 @@ mod tests { ); } } - let envelope = concat!( - r#"WDL-ENC:{"v":1,"alg":"AES-256-GCM","kid":"local:test:secret-envelope:v1","#, - r#""edek":"ICEiIyQlJicoKSorrmiFFNwSNL789zDTZysVGjsTpksuXqulg0Mt7JrBiLTG0wE1moqP31Jd+edymdiU","#, - r#""iv":"LC0uLzAxMjM0NTY3","ct":"IZhLZQvljbpT5euHV5YP","tag":"0wuyjfzeZfuY5hD3HRoKdg=="}"# - ); let out = decryptor .decrypt_hash_entries( - "secrets:demo:api", - vec![("TOKEN".to_string(), envelope.as_bytes().to_vec())], + &vector.hash_key, + vec![( + vector.field_name.clone(), + vector.envelope.as_bytes().to_vec(), + )], ) .unwrap(); - assert_eq!(out.get("TOKEN").unwrap(), "sensitive-value"); + assert_eq!(out.get(&vector.field_name), Some(&vector.plaintext)); let metrics = decryptor.metrics.render_prometheus(); assert!(metrics.contains(&format!( r#"wdl_secret_dek_cache_evictions_total{{reason="expired",service="redis-proxy"}} {DEK_CACHE_SHARD_LIMIT}"# diff --git a/rust/scheduler/src/cron/dispatch.rs b/rust/scheduler/src/cron/dispatch.rs index 24ce4e1..7fd6aee 100644 --- a/rust/scheduler/src/cron/dispatch.rs +++ b/rust/scheduler/src/cron/dispatch.rs @@ -7,8 +7,8 @@ use crate::{ redis_fields_with_error, scheduler_fields_with_error, }; -use super::reference::{RefVerdict, classify_ref, parse_ref}; -use super::slot::{lease_key, next_fire_ms, slot_key, slot_ms_for}; +use super::reference::{CronEntry, RefVerdict, classify_ref, cron_worker_key, parse_ref}; +use super::slot::{lease_key, next_fire_ms, slot_expire_at, slot_key, slot_ms_for}; const CRON_CLAIM_ADVANCE_SCRIPT: &str = r#" if redis.call('SET', KEYS[2], ARGV[1], 'NX', 'EX', ARGV[2]) then @@ -30,7 +30,7 @@ async fn claim_and_advance_ref( let from_key = slot_key(from_slot); let lease = lease_key(from_slot, &reference); let to_key = slot_key(to_slot); - let to_expire_at = to_slot / 1000 + 600; + let to_expire_at = slot_expire_at(to_slot); let instance_id = state.instance_id.clone(); let lease_ttl_s = state.config.lease_ttl_s; let lease_ttl_s_arg = lease_ttl_s.to_string(); @@ -70,6 +70,30 @@ async fn remove_ref_from_slot( Ok(()) } +#[derive(Debug, PartialEq, Eq)] +enum CronDispatchDecision { + RemoveInvalidRef { error_message: String }, + AdvanceOnly { next_slot: i64 }, + Fire { next_slot: i64 }, +} + +fn cron_dispatch_decision(entry: &CronEntry, slot_ms: i64, tick_now: i64) -> CronDispatchDecision { + let next_fire = match next_fire_ms(&entry.cron, &entry.timezone, tick_now) { + Ok(ms) => ms, + Err(err) => { + return CronDispatchDecision::RemoveInvalidRef { + error_message: err.message, + }; + } + }; + let next_slot = slot_ms_for(next_fire); + if slot_ms < slot_ms_for(tick_now) { + CronDispatchDecision::AdvanceOnly { next_slot } + } else { + CronDispatchDecision::Fire { next_slot } + } +} + async fn process_ref( state: AppState, reference: String, @@ -83,7 +107,7 @@ async fn process_ref( remove_ref_from_slot(&state, slot_ms, &reference).await?; return Ok(()); }; - let cron_hash_key = format!("crons:{}:{}", parts.ns, parts.worker); + let cron_hash_key = cron_worker_key(&parts.ns, &parts.worker); let cron_id = parts.cron_id.clone(); let (meta_str, entry_str): (Option, Option) = state .redis @@ -139,10 +163,9 @@ async fn process_ref( // moved to next_slot, giving at-most-once per slot. // (c) stranded refs (slot_ms < current_slot) advance but do NOT fire // — preserves CF's "skip missed after outage" semantic. - let current_slot = slot_ms_for(tick_now); - let next_fire = match next_fire_ms(&entry.cron, &entry.timezone, tick_now) { - Ok(ms) => ms, - Err(err) => { + let decision = cron_dispatch_decision(&entry, slot_ms, tick_now); + let next_slot = match &decision { + CronDispatchDecision::RemoveInvalidRef { error_message } => { state .metrics .increment("cron_stale_refs_cleaned", &[("service", SERVICE)], 1.0); @@ -157,14 +180,15 @@ async fn process_ref( "gen": parts.r#gen, "slot": slot_ms, "reason": "next_fire_failed", - "error_message": err.message, + "error_message": error_message, }), ); remove_ref_from_slot(&state, slot_ms, &reference).await?; return Ok(()); } + CronDispatchDecision::AdvanceOnly { next_slot } + | CronDispatchDecision::Fire { next_slot } => *next_slot, }; - let next_slot = slot_ms_for(next_fire); if !claim_and_advance_ref(&state, &reference, slot_ms, next_slot).await? { state.metrics.increment( "cron_fires", @@ -185,7 +209,7 @@ async fn process_ref( return Ok(()); } - if slot_ms < current_slot { + if matches!(decision, CronDispatchDecision::AdvanceOnly { .. }) { state.metrics.increment( "cron_fires", &[("service", SERVICE), ("outcome", "skipped_stale")], @@ -201,7 +225,7 @@ async fn process_ref( "cron_id": parts.cron_id, "from_slot": slot_ms, "to_slot": next_slot, - "lag_ms": current_slot - slot_ms, + "lag_ms": slot_ms_for(tick_now) - slot_ms, }), ); return Ok(()); @@ -314,40 +338,45 @@ pub(crate) async fn tick(state: AppState) -> SchedulerResult<()> { #[cfg(test)] mod tests { - #[test] - fn invalid_next_fire_refs_are_removed_instead_of_bubbling_error() { - let source = include_str!("dispatch.rs"); - let production_source = source - .split("#[cfg(test)]") - .next() - .expect("dispatch source must include production section"); + use super::*; - assert!(production_source.contains( - "let next_fire = match next_fire_ms(&entry.cron, &entry.timezone, tick_now)" - )); - assert!( - !production_source.contains("next_fire_ms(&entry.cron, &entry.timezone, tick_now)?") - ); - assert!(production_source.contains(r#""reason": "next_fire_failed""#)); - assert!( - production_source.contains("remove_ref_from_slot(&state, slot_ms, &reference).await?") + fn cron_entry(cron: &str) -> CronEntry { + CronEntry { + cron: cron.to_string(), + timezone: "UTC".to_string(), + r#gen: 1, + } + } + + #[test] + fn invalid_next_fire_returns_remove_ref_decision() { + let decision = cron_dispatch_decision( + &cron_entry("not a cron"), + 1_700_000_000_000, + 1_700_000_000_000, ); + + match decision { + CronDispatchDecision::RemoveInvalidRef { error_message } => { + assert!(!error_message.is_empty()); + } + other => panic!("expected invalid next-fire drop, got {other:?}"), + } } #[test] - fn previous_slot_refs_advance_without_runtime_fire() { - let source = include_str!("dispatch.rs"); - let claim_pos = source - .find("claim_and_advance_ref(&state, &reference, slot_ms, next_slot)") - .expect("cron dispatch must claim and advance refs before firing"); - let stale_skip_pos = source - .find("if slot_ms < current_slot") - .expect("cron dispatch must skip firing stale lookback slots"); - let runtime_post_pos = source - .find("post_runtime(") - .expect("cron dispatch must post runtime after stale-slot checks"); + fn previous_slot_returns_advance_only_decision() { + let tick_now = 1_700_000_000_000; + let current_slot = slot_ms_for(tick_now); + let entry = cron_entry("*/5 * * * *"); - assert!(claim_pos < stale_skip_pos); - assert!(stale_skip_pos < runtime_post_pos); + assert!(matches!( + cron_dispatch_decision(&entry, current_slot - 60_000, tick_now), + CronDispatchDecision::AdvanceOnly { .. } + )); + assert!(matches!( + cron_dispatch_decision(&entry, current_slot, tick_now), + CronDispatchDecision::Fire { .. } + )); } } diff --git a/rust/scheduler/src/cron/reference.rs b/rust/scheduler/src/cron/reference.rs index 4038f36..3c2ad57 100644 --- a/rust/scheduler/src/cron/reference.rs +++ b/rust/scheduler/src/cron/reference.rs @@ -1,4 +1,11 @@ use serde::Deserialize; +use wdl_rust_common::{ + identity::{is_valid_route_ns, is_valid_worker_name}, + version::parse_version_tag, +}; + +const CRON_WORKER_KEY_PREFIX: &str = "crons:"; +pub(crate) const CRON_WORKER_KEY_SCAN_PATTERN: &str = "crons:*:*"; #[derive(Clone, Debug)] pub(crate) struct RefParts { @@ -29,6 +36,18 @@ pub(crate) enum RefVerdict { Corrupt, } +pub(crate) fn cron_worker_key(ns: &str, worker: &str) -> String { + format!("{CRON_WORKER_KEY_PREFIX}{ns}:{worker}") +} + +pub(crate) fn parse_cron_worker_key(key: &str) -> Option<(&str, &str)> { + let (ns, worker) = key.strip_prefix(CRON_WORKER_KEY_PREFIX)?.split_once(':')?; + if !is_valid_route_ns(ns) || !is_valid_worker_name(worker) { + return None; + } + Some((ns, worker)) +} + pub(crate) fn ref_for(ns: &str, worker: &str, cron_id: &str, r#gen: i64) -> String { format!("{ns}:{worker}:{cron_id}:{gen}") } @@ -41,6 +60,9 @@ pub(crate) fn parse_ref(reference: &str) -> Option { if parts.iter().any(|part| part.is_empty()) { return None; } + if !is_valid_route_ns(parts[0]) || !is_valid_worker_name(parts[1]) { + return None; + } let r#gen = parts[3].parse::().ok()?; Some(RefParts { ns: parts[0].to_string(), @@ -50,6 +72,15 @@ pub(crate) fn parse_ref(reference: &str) -> Option { }) } +fn validated_cron_meta_version(meta: CronMeta) -> Option { + parse_version_tag(&meta.version).ok()?; + Some(meta.version) +} + +pub(crate) fn cron_meta_version(raw: &str) -> Option { + validated_cron_meta_version(serde_json::from_str::(raw).ok()?) +} + pub(crate) fn classify_ref( parts: &RefParts, meta_str: Option, @@ -67,12 +98,15 @@ pub(crate) fn classify_ref( let Ok(entry) = serde_json::from_str::(&entry_str) else { return RefVerdict::Corrupt; }; + let Some(active_version) = validated_cron_meta_version(meta) else { + return RefVerdict::Corrupt; + }; if entry.r#gen != parts.r#gen { return RefVerdict::Stale("gen_mismatch"); } RefVerdict::Fire { entry, - active_version: meta.version, + active_version, } } @@ -81,6 +115,36 @@ mod tests { use serde_json::json; use super::*; + use crate::test_fixtures::{scheduler_projection_contract, version_tag_cases}; + + #[test] + fn cron_projection_contract_matches_control_writer() { + let fixture = scheduler_projection_contract(); + let cron = fixture.cron; + + assert_eq!(cron_worker_key(&cron.ns, &cron.worker), cron.worker_key); + assert_eq!( + parse_cron_worker_key(&cron.worker_key), + Some((cron.ns.as_str(), cron.worker.as_str())) + ); + assert_eq!( + ref_for(&cron.ns, &cron.worker, &cron.cron_id, cron.r#gen), + cron.reference + ); + let parts = parse_ref(&cron.reference).expect("fixture cron reference must parse"); + match classify_ref(&parts, Some(cron.meta.json), Some(cron.entry.json)) { + RefVerdict::Fire { + entry, + active_version, + } => { + assert_eq!(active_version, cron.meta.version); + assert_eq!(entry.cron, cron.entry.cron); + assert_eq!(entry.timezone, cron.entry.timezone); + assert_eq!(entry.r#gen, cron.entry.r#gen); + } + _ => panic!("fixture cron projection must be fireable"), + } + } #[test] fn parse_ref_round_trip() { @@ -103,6 +167,21 @@ mod tests { assert!(parse_ref("demo:hello::7").is_none()); assert!(parse_ref("demo:hello:abc123:x").is_none()); assert!(parse_ref("demo:hello:abc123:").is_none()); + assert!(parse_ref("__platform__:hello:abc123:7").is_none()); + assert!(parse_ref("Demo:hello:abc123:7").is_none()); + assert!(parse_ref("demo:/bad:abc123:7").is_none()); + } + + #[test] + fn parse_cron_worker_key_rejects_non_worker_hash_keys() { + assert_eq!(parse_cron_worker_key("queue:demo:jobs"), None); + assert_eq!(parse_cron_worker_key("crons:demo"), None); + assert_eq!(parse_cron_worker_key("crons::worker"), None); + assert_eq!(parse_cron_worker_key("crons:demo:"), None); + assert_eq!(parse_cron_worker_key("crons:demo:worker:extra"), None); + assert_eq!(parse_cron_worker_key("crons:__platform__:worker"), None); + assert_eq!(parse_cron_worker_key("crons:Demo:worker"), None); + assert_eq!(parse_cron_worker_key("crons:demo:/bad"), None); } #[test] @@ -130,6 +209,29 @@ mod tests { } } + #[test] + fn classify_ref_versions_match_cross_language_fixture() { + let parts = RefParts { + ns: "demo".to_string(), + worker: "hello".to_string(), + cron_id: "abc".to_string(), + r#gen: 3, + }; + let entry = Some(json!({ "cron": "*/5 * * * *", "timezone": "UTC", "gen": 3 }).to_string()); + for (version, valid) in version_tag_cases() { + let verdict = classify_ref( + &parts, + Some(json!({ "version": version }).to_string()), + entry.clone(), + ); + assert_eq!( + matches!(verdict, RefVerdict::Fire { .. }), + valid, + "{version:?}" + ); + } + } + #[test] fn classify_ref_marks_missing_or_mismatched_state_stale() { let parts = RefParts { diff --git a/rust/scheduler/src/cron/slot.rs b/rust/scheduler/src/cron/slot.rs index d465964..cd91f6d 100644 --- a/rust/scheduler/src/cron/slot.rs +++ b/rust/scheduler/src/cron/slot.rs @@ -14,6 +14,10 @@ pub(crate) fn slot_key(slot_ms: i64) -> String { format!("cron-slot:{slot_ms}") } +pub(crate) fn slot_expire_at(slot_ms: i64) -> i64 { + slot_ms.div_euclid(1000) + 600 +} + pub(crate) fn wait_ms_until_next_slot(now_ms: i64) -> u64 { (slot_ms_for(now_ms) + 60_000 - now_ms).max(1) as u64 } @@ -45,6 +49,7 @@ mod tests { use super::*; use crate::cron::reference::ref_for; + use crate::test_fixtures::scheduler_projection_contract; #[derive(serde::Deserialize)] #[serde(rename_all = "camelCase")] @@ -60,8 +65,13 @@ mod tests { #[test] fn cron_key_builders_compose_scheduler_keys() { + let fixture = scheduler_projection_contract(); + assert_eq!(slot_key(fixture.cron.slot_ms), fixture.cron.slot_key); + assert_eq!( + slot_expire_at(fixture.cron.slot_ms), + fixture.cron.slot_expire_at + ); let reference = ref_for("demo", "hello", "abc123", 7); - assert_eq!(slot_key(1_710_000_000_000), "cron-slot:1710000000000"); assert_eq!( lease_key(1_710_000_000_000, &reference), "cron-lease:1710000000000:demo:hello:abc123:7" diff --git a/rust/scheduler/src/cron/sweep.rs b/rust/scheduler/src/cron/sweep.rs index 7acc5f6..84263ac 100644 --- a/rust/scheduler/src/cron/sweep.rs +++ b/rust/scheduler/src/cron/sweep.rs @@ -6,8 +6,10 @@ use crate::{ AppState, LogLevel, SERVICE, SchedulerResult, indexed_keys_after_backfill, log, now_ms, }; -use super::reference::{CronEntry, ref_for}; -use super::slot::{next_fire_ms, slot_key, slot_ms_for}; +use super::reference::{ + CRON_WORKER_KEY_SCAN_PATTERN, CronEntry, cron_meta_version, parse_cron_worker_key, ref_for, +}; +use super::slot::{next_fire_ms, slot_expire_at, slot_key, slot_ms_for}; const CRON_WORKER_INDEX_KEY: &str = "cron:index:workers"; const CRON_WORKER_INDEX_BACKFILLED_KEY: &str = "cron:index:workers:backfilled"; @@ -19,20 +21,11 @@ async fn cron_worker_keys(state: &AppState) -> Result, redis::RedisE state, CRON_WORKER_INDEX_KEY, CRON_WORKER_INDEX_BACKFILLED_KEY, - "crons:*:*", + CRON_WORKER_KEY_SCAN_PATTERN, ) .await } -fn parse_cron_worker_key(key: &str) -> Option<(&str, &str)> { - let parts = key.split(':').collect::>(); - if parts.len() == 3 && parts[0] == "crons" && !parts[1].is_empty() && !parts[2].is_empty() { - Some((parts[1], parts[2])) - } else { - None - } -} - async fn fetch_cron_hash_chunk( state: &AppState, keys: &[String], @@ -65,7 +58,7 @@ fn cron_slot_writes(slot_refs: BTreeMap>) -> Vec slot, key: slot_key(slot), refs, - expire_at: slot / 1000 + 600, + expire_at: slot_expire_at(slot), }) .collect() } @@ -78,6 +71,23 @@ fn count_sadd_replies(replies: &[i64]) -> u64 { .sum() } +fn has_dispatchable_cron_meta(hash: &HashMap) -> bool { + hash.get("__meta__") + .and_then(|raw| cron_meta_version(raw)) + .is_some() +} + +fn dispatchable_cron_worker<'a>( + key: &'a str, + hash: &HashMap, +) -> Result<(&'a str, &'a str), &'static str> { + let (ns, worker) = parse_cron_worker_key(key).ok_or("invalid_identity")?; + if !has_dispatchable_cron_meta(hash) { + return Err("invalid_meta"); + } + Ok((ns, worker)) +} + async fn write_cron_slot_refs( state: &AppState, slot_refs: BTreeMap>, @@ -107,17 +117,25 @@ pub(crate) async fn sweep(state: AppState) -> SchedulerResult<()> { let started = now_ms(); let mut workers = 0_u64; let mut skipped = 0_u64; + let mut skipped_workers = 0_u64; let keys = cron_worker_keys(&state).await?; let mut slot_refs: BTreeMap> = BTreeMap::new(); for chunk in keys.chunks(CRON_SWEEP_READ_CHUNK_SIZE) { let hashes = fetch_cron_hash_chunk(&state, chunk).await?; for (key, hash) in chunk.iter().zip(hashes) { workers += 1; - if !hash.contains_key("__meta__") { - continue; - } - let Some((ns, worker)) = parse_cron_worker_key(key) else { - continue; + let (ns, worker) = match dispatchable_cron_worker(key, &hash) { + Ok(identity) => identity, + Err(reason) => { + skipped_workers += 1; + log( + &state, + LogLevel::Warn, + "cron_sweep_worker_skipped", + json!({ "worker_key": key, "reason": reason }), + ); + continue; + } }; let now = now_ms(); for (id, raw) in hash { @@ -177,6 +195,11 @@ pub(crate) async fn sweep(state: AppState) -> SchedulerResult<()> { &[("service", SERVICE)], skipped as f64, ); + state.metrics.increment( + "cron_sweep_workers_skipped", + &[("service", SERVICE)], + skipped_workers as f64, + ); log( &state, LogLevel::Info, @@ -185,6 +208,8 @@ pub(crate) async fn sweep(state: AppState) -> SchedulerResult<()> { "workers": workers, "re_added": re_added, "skipped": skipped, + "entries_skipped": skipped, + "workers_skipped": skipped_workers, "duration_ms": now_ms() - started, }), ); @@ -194,22 +219,12 @@ pub(crate) async fn sweep(state: AppState) -> SchedulerResult<()> { #[cfg(test)] mod tests { use super::*; - - #[test] - fn parse_cron_worker_key_accepts_only_scheduler_cron_hashes() { - assert_eq!( - parse_cron_worker_key("crons:demo:worker"), - Some(("demo", "worker")) - ); - assert_eq!(parse_cron_worker_key("queue:demo:jobs"), None); - assert_eq!(parse_cron_worker_key("crons:demo"), None); - assert_eq!(parse_cron_worker_key("crons::worker"), None); - assert_eq!(parse_cron_worker_key("crons:demo:"), None); - } + use crate::test_fixtures::scheduler_projection_contract; #[test] fn cron_worker_index_keys_are_stable() { - assert_eq!(CRON_WORKER_INDEX_KEY, "cron:index:workers"); + let fixture = scheduler_projection_contract(); + assert_eq!(CRON_WORKER_INDEX_KEY, fixture.cron.worker_index_key); assert_eq!( CRON_WORKER_INDEX_BACKFILLED_KEY, "cron:index:workers:backfilled" @@ -248,4 +263,41 @@ mod tests { fn count_sadd_replies_ignores_expireat_replies() { assert_eq!(count_sadd_replies(&[2, 1, 0, 1, 4, 1]), 6); } + + #[test] + fn cron_sweep_requires_dispatchable_worker_metadata() { + let valid = HashMap::from([( + "__meta__".to_string(), + json!({ "version": "v1" }).to_string(), + )]); + assert_eq!( + dispatchable_cron_worker("crons:demo:worker", &valid), + Ok(("demo", "worker")) + ); + assert_eq!( + dispatchable_cron_worker("crons:__platform__:worker", &valid), + Err("invalid_identity") + ); + assert_eq!( + dispatchable_cron_worker("crons:demo:worker", &HashMap::new()), + Err("invalid_meta") + ); + assert_eq!( + dispatchable_cron_worker( + "crons:demo:worker", + &HashMap::from([("__meta__".to_string(), "{broken".to_string(),)]) + ), + Err("invalid_meta") + ); + assert_eq!( + dispatchable_cron_worker( + "crons:demo:worker", + &HashMap::from([( + "__meta__".to_string(), + json!({ "version": "v9007199254740992" }).to_string(), + )]) + ), + Err("invalid_meta") + ); + } } diff --git a/rust/scheduler/src/lib.rs b/rust/scheduler/src/lib.rs index 5de9227..cbc8e2e 100644 --- a/rust/scheduler/src/lib.rs +++ b/rust/scheduler/src/lib.rs @@ -13,6 +13,8 @@ mod runtime_client; mod server; mod state; mod tasks; +#[cfg(test)] +mod test_fixtures; mod time; mod workflows; diff --git a/rust/scheduler/src/queue/delayed.rs b/rust/scheduler/src/queue/delayed.rs index 678dc72..fd554ff 100644 --- a/rust/scheduler/src/queue/delayed.rs +++ b/rust/scheduler/src/queue/delayed.rs @@ -6,6 +6,7 @@ use redis::streams::StreamReadReply; use serde_json::json; use tokio::time::sleep; use wdl_rust_common::hash::fnv1a64; +use wdl_rust_common::queue_keys::{QUEUE_DELAYED_WAKE_KEY_FIELD, QUEUE_DELAYED_WAKE_STREAM}; use wdl_rust_common::redis_eval::append_eval_cmd; use crate::{ @@ -17,7 +18,6 @@ use super::{ parse_delayed_key, queue_orphaned_key, queue_stream_key, resolve_consumer, stream_id_to_entry, }; -pub(crate) const QUEUE_DELAYED_WAKE_STREAM: &str = "queue-delayed-wake"; const QUEUE_DELAYED_CLAIM_SAFETY_MS: u64 = 5_000; const QUEUE_DELAYED_NO_PROGRESS_BACKOFF_MS: u64 = 100; const QUEUE_DELAYED_WAKE_RETRY_BASE_MS: u64 = 1_000; @@ -500,7 +500,7 @@ pub(crate) async fn queue_delayed_wake_loop(state: AppState) -> SchedulerResult< for id in key.ids { last_id = id.id.clone(); let entry = stream_id_to_entry(id); - let Some(delayed_key) = entry.fields.get("delayed_key") else { + let Some(delayed_key) = entry.fields.get(QUEUE_DELAYED_WAKE_KEY_FIELD) else { continue; }; if parse_delayed_key(delayed_key).is_none() { diff --git a/rust/scheduler/src/queue/keys.rs b/rust/scheduler/src/queue/keys.rs index b4f1748..001de6f 100644 --- a/rust/scheduler/src/queue/keys.rs +++ b/rust/scheduler/src/queue/keys.rs @@ -1,60 +1,6 @@ pub(crate) use wdl_rust_common::queue_keys::{ - QUEUE_CONSUMER_INDEX_KEY, QUEUE_DELAYED_INDEX_KEY, QUEUE_STREAM_INDEX_KEY, parse_consumer_key, - parse_delayed_key, parse_stream_key, queue_consumer_key, queue_delayed_key, queue_dlq_key, - queue_orphaned_key, queue_stream_key, + QUEUE_CONSUMER_INDEX_KEY, QUEUE_CONSUMER_SCAN_PATTERN, QUEUE_DELAYED_INDEX_KEY, + QUEUE_DELAYED_SCAN_PATTERN, QUEUE_STREAM_INDEX_KEY, QUEUE_STREAM_SCAN_PATTERN, + parse_consumer_key, parse_delayed_key, parse_stream_key, queue_consumer_key, queue_delayed_key, + queue_dlq_key, queue_orphaned_key, queue_stream_key, }; - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn queue_key_builders_compose_redis_key_shapes() { - assert_eq!(QUEUE_CONSUMER_INDEX_KEY, "queue:index:consumers"); - assert_eq!(QUEUE_STREAM_INDEX_KEY, "queue:index:streams"); - assert_eq!(QUEUE_DELAYED_INDEX_KEY, "queue:index:delayed"); - assert_eq!(queue_stream_key("demo", "jobs"), "queue:demo:jobs:s"); - assert_eq!(queue_delayed_key("demo", "jobs"), "queue-delayed:demo:jobs"); - assert_eq!(queue_dlq_key("demo", "jobs"), "queue:demo:jobs:dlq"); - assert_eq!( - queue_orphaned_key("demo", "jobs"), - "queue-orphaned:demo:jobs" - ); - assert_eq!( - queue_consumer_key("demo", "jobs"), - "queue-consumer:demo:jobs" - ); - assert_eq!(queue_stream_key("t", "my-queue-1"), "queue:t:my-queue-1:s"); - } - - #[test] - fn parse_queue_keys() { - assert_eq!( - parse_stream_key("queue:demo:jobs:s"), - Some(("demo".to_string(), "jobs".to_string())) - ); - assert_eq!( - parse_delayed_key("queue-delayed:demo:jobs"), - Some(("demo".to_string(), "jobs".to_string())) - ); - assert_eq!( - parse_consumer_key("queue-consumer:demo:jobs"), - Some(("demo".to_string(), "jobs".to_string())) - ); - assert_eq!( - parse_stream_key("queue:t:my-queue-1:s"), - Some(("t".to_string(), "my-queue-1".to_string())) - ); - assert_eq!(parse_stream_key("queue:demo:jobs"), None); - assert_eq!(parse_stream_key("queue-delayed:demo:jobs"), None); - assert_eq!(parse_stream_key("queue::jobs:s"), None); - assert_eq!(parse_stream_key("notaqueue:demo:jobs:s"), None); - assert_eq!(parse_stream_key(""), None); - assert_eq!(parse_delayed_key("queue:demo:jobs:s"), None); - assert_eq!(parse_delayed_key("queue-delayed::jobs"), None); - assert_eq!(parse_delayed_key("queue-delayed:demo:"), None); - assert_eq!(parse_consumer_key("queue-consumer:demo"), None); - assert_eq!(parse_consumer_key("queue-consumer:"), None); - assert_eq!(parse_consumer_key("other:demo:jobs"), None); - } -} diff --git a/rust/scheduler/src/queue/registry.rs b/rust/scheduler/src/queue/registry.rs index a74a6b5..94db856 100644 --- a/rust/scheduler/src/queue/registry.rs +++ b/rust/scheduler/src/queue/registry.rs @@ -3,21 +3,26 @@ use std::future::Future; use std::sync::Arc; use redis::{AsyncCommands, Value}; +use wdl_rust_common::identity::{is_valid_route_ns, is_valid_worker_name}; +use wdl_rust_common::queue_keys::is_valid_queue_name; +use wdl_rust_common::version::parse_version_tag; use crate::{ - AppState, CONSUMER_GROUP, QueueState, SchedulerResult, indexed_data_keys, indexed_keys, + AppState, CONSUMER_GROUP, MAX_BATCH_SIZE_CAP, QueueState, SchedulerResult, indexed_data_keys, + indexed_keys, }; use super::{ - Consumer, QUEUE_CONSUMER_INDEX_KEY, QUEUE_DELAYED_INDEX_KEY, QUEUE_STREAM_INDEX_KEY, + Consumer, QUEUE_CONSUMER_INDEX_KEY, QUEUE_CONSUMER_SCAN_PATTERN, QUEUE_DELAYED_INDEX_KEY, + QUEUE_DELAYED_SCAN_PATTERN, QUEUE_STREAM_INDEX_KEY, QUEUE_STREAM_SCAN_PATTERN, parse_consumer_key, queue_consumer_key, queue_stream_key, redis_error_is_busygroup, }; const QUEUE_RECONCILE_CONSUMER_HASH_BATCH_SIZE: usize = 128; -pub(crate) fn finite_or(value: Option<&String>, fallback: i64) -> i64 { +fn finite_or(value: Option<&String>, fallback: i64) -> i64 { value - .and_then(|v| v.parse::().ok()) + .and_then(|value| value.parse::().ok()) .unwrap_or(fallback) } @@ -26,15 +31,25 @@ pub(crate) fn hydrate_consumer( queue: &str, hash: &HashMap, ) -> Option { - let worker = hash.get("worker")?.clone(); - let version = hash.get("version")?.clone(); - let max_batch_size = finite_or(hash.get("max_batch_size"), 10).max(1) as usize; + if !is_valid_route_ns(ns) || !is_valid_queue_name(queue) { + return None; + } + let worker = hash + .get("worker") + .filter(|value| is_valid_worker_name(value)) + .cloned()?; + let version = hash + .get("version") + .filter(|value| parse_version_tag(value).is_ok()) + .cloned()?; + let max_batch_size = + finite_or(hash.get("max_batch_size"), 10).clamp(1, MAX_BATCH_SIZE_CAP as i64) as usize; let max_batch_timeout_ms = finite_or(hash.get("max_batch_timeout_ms"), 5000); let max_retries = finite_or(hash.get("max_retries"), 3); let retry_delay_secs = finite_or(hash.get("retry_delay_secs"), 0).max(0); let dead_letter_queue = hash .get("dead_letter_queue") - .filter(|v| !v.is_empty()) + .filter(|value| !value.is_empty()) .cloned(); Some(Consumer { ns: ns.to_string(), @@ -72,8 +87,12 @@ async fn flush_indexed_streams( pub(crate) async fn queue_reconcile(state: AppState) -> SchedulerResult<()> { let mut seen = HashSet::new(); let mut registry_changed = false; - let consumer_keys = - indexed_keys(&state, QUEUE_CONSUMER_INDEX_KEY, "queue-consumer:*:*").await?; + let consumer_keys = indexed_keys( + &state, + QUEUE_CONSUMER_INDEX_KEY, + QUEUE_CONSUMER_SCAN_PATTERN, + ) + .await?; let mut indexed_streams = Vec::new(); for consumer_key_chunk in consumer_keys.chunks(QUEUE_RECONCILE_CONSUMER_HASH_BATCH_SIZE) { let consumer_hashes: Vec> = state @@ -148,13 +167,15 @@ pub(crate) async fn queue_reconcile(state: AppState) -> SchedulerResult<()> { // strand a stream outside the consumer loop. refresh_consumer_streams(&state).await; - let streams = indexed_data_keys(&state, QUEUE_STREAM_INDEX_KEY, "queue:*:*:s").await?; + let streams = + indexed_data_keys(&state, QUEUE_STREAM_INDEX_KEY, QUEUE_STREAM_SCAN_PATTERN).await?; { let mut known = state.queues.known_streams.write().await; known.clear(); known.extend(streams); } - let delayed = indexed_data_keys(&state, QUEUE_DELAYED_INDEX_KEY, "queue-delayed:*:*").await?; + let delayed = + indexed_data_keys(&state, QUEUE_DELAYED_INDEX_KEY, QUEUE_DELAYED_SCAN_PATTERN).await?; let delayed_changed = { let delayed = delayed.into_iter().collect::>(); let mut known = state.queues.known_delayed.write().await; @@ -176,6 +197,9 @@ pub(crate) async fn refresh_consumer_streams(state: &AppState) { } async fn refresh_consumer_streams_for(queues: &QueueState) { + // Lock order is registry -> consumer_streams. Readers clone consumer_streams + // before consulting registry, so no path holds consumer_streams while waiting + // for registry. let registry = queues.registry.read().await; let snapshot = sorted_consumer_streams(®istry); *queues.consumer_streams.write().await = snapshot; @@ -216,10 +240,7 @@ where let hash = load_hash(queue_consumer_key(ns, queue)).await?; let consumer = hydrate_consumer(ns, queue, &hash); write_resolved_consumer(queues, stream_key, consumer.as_ref()).await; - let Some(consumer) = consumer else { - return Ok(None); - }; - Ok(Some(consumer)) + Ok(consumer) } async fn write_resolved_consumer( @@ -236,14 +257,17 @@ async fn write_resolved_consumer( } else { queues.registry.write().await.remove(stream_key) }; - if previous.as_ref() != consumer { + let registry_changed = previous.as_ref() != consumer; + if registry_changed { refresh_consumer_streams_for(queues).await; + queues.delayed_changed.notify_one(); } } #[cfg(test)] mod tests { use super::*; + use crate::test_fixtures::scheduler_projection_contract; use std::sync::{Arc as StdArc, Mutex}; fn str_map(items: &[(&str, &str)]) -> HashMap { @@ -253,6 +277,16 @@ mod tests { .collect() } + fn valid_consumer_hash(worker: &str, version: &str) -> HashMap { + str_map(&[ + ("worker", worker), + ("version", version), + ("max_batch_size", "10"), + ("max_batch_timeout_ms", "5000"), + ("max_retries", "3"), + ]) + } + #[test] fn queue_index_lookup_plan_covers_backfill_and_stale_branches() { assert_eq!( @@ -296,7 +330,7 @@ mod tests { } #[test] - fn hydrate_consumer_preserves_defaults_and_explicit_zeroes() { + fn hydrate_consumer_preserves_defaults_and_runtime_batch_cap() { let defaults = hydrate_consumer( "demo", "jobs", @@ -336,48 +370,55 @@ mod tests { &str_map(&[ ("worker", "w"), ("version", "v3"), + ("max_batch_size", "1000"), ("max_retries", "0"), ("max_batch_timeout_ms", "0"), ("retry_delay_secs", "0"), ]), ) .unwrap(); + assert_eq!(zeroes.max_batch_size, MAX_BATCH_SIZE_CAP); assert_eq!(zeroes.max_retries, 0); assert_eq!(zeroes.max_batch_timeout_ms, 0); assert_eq!(zeroes.retry_delay_secs, 0); } #[test] - fn hydrate_consumer_falls_back_for_invalid_or_incomplete_hashes() { - let invalid = hydrate_consumer( - "demo", - "jobs", - &str_map(&[ - ("worker", "w"), - ("version", "v1"), - ("max_batch_size", "bogus"), - ("max_retries", "nan"), - ("retry_delay_secs", "bogus"), - ]), - ) - .unwrap(); - assert_eq!(invalid.max_batch_size, 10); - assert_eq!(invalid.max_retries, 3); - assert_eq!(invalid.retry_delay_secs, 0); + fn hydrate_consumer_matches_control_projection_fixture() { + let fixture = scheduler_projection_contract(); + let contract = fixture.queue_consumer; + let consumer = hydrate_consumer(&contract.ns, &contract.queue, &contract.fields) + .expect("fixture queue consumer projection must hydrate"); + assert_eq!( + consumer.worker_id, + format!("{}:{}:{}", contract.ns, contract.worker, contract.version) + ); + assert_eq!(consumer.max_batch_size, 12); + assert_eq!(consumer.max_batch_timeout_ms, 250); + assert_eq!(consumer.max_retries, 4); + assert_eq!(consumer.retry_delay_secs, 17); + assert_eq!(consumer.dead_letter_queue.as_deref(), Some("jobs-dlq")); + } + + #[test] + fn hydrate_consumer_rejects_noncanonical_dispatch_identity() { + assert!(hydrate_consumer("demo", "jobs", &HashMap::new()).is_none()); assert!(hydrate_consumer("demo", "jobs", &str_map(&[("version", "v1")])).is_none()); assert!(hydrate_consumer("demo", "jobs", &str_map(&[("worker", "w")])).is_none()); + assert!( + hydrate_consumer("__platform__", "jobs", &valid_consumer_hash("w", "v1")).is_none() + ); + assert!( + hydrate_consumer("demo", "jobs", &valid_consumer_hash("bad:worker", "v1")).is_none() + ); + assert!(hydrate_consumer("demo", "jobs", &valid_consumer_hash("w", "v01")).is_none()); } #[test] fn sorted_consumer_streams_returns_stable_key_snapshot() { let mut registry = HashMap::new(); - let jobs = hydrate_consumer( - "demo", - "jobs", - &str_map(&[("worker", "w"), ("version", "v1")]), - ) - .unwrap(); + let jobs = hydrate_consumer("demo", "jobs", &valid_consumer_hash("w", "v1")).unwrap(); registry.insert(queue_stream_key("demo", "z"), jobs.clone()); registry.insert(queue_stream_key("demo", "a"), jobs); @@ -392,12 +433,8 @@ mod tests { async fn resolve_consumer_loads_authoritative_hash_and_refreshes_registry() { let queues = QueueState::default(); let stream_key = queue_stream_key("demo", "jobs"); - let stale = hydrate_consumer( - "demo", - "jobs", - &str_map(&[("worker", "old-worker"), ("version", "v1")]), - ) - .unwrap(); + let stale = + hydrate_consumer("demo", "jobs", &valid_consumer_hash("old-worker", "v1")).unwrap(); queues .registry .write() @@ -410,6 +447,8 @@ mod tests { ("worker", "fresh-worker"), ("version", "v2"), ("max_batch_size", "4"), + ("max_batch_timeout_ms", "5000"), + ("max_retries", "3"), ]); let consumer = resolve_consumer_with_hash_loader(&queues, &stream_key, "demo", "jobs", { let loaded_keys = loaded_keys.clone(); @@ -448,31 +487,35 @@ mod tests { } #[tokio::test] - async fn resolve_consumer_removes_stale_registry_when_authoritative_hash_is_missing() { - let queues = QueueState::default(); - let stream_key = queue_stream_key("demo", "jobs"); - let stale = hydrate_consumer( - "demo", - "jobs", - &str_map(&[("worker", "old-worker"), ("version", "v1")]), - ) - .unwrap(); - queues - .registry - .write() - .await - .insert(stream_key.clone(), stale); - refresh_consumer_streams_for(&queues).await; - - let consumer = - resolve_consumer_with_hash_loader(&queues, &stream_key, "demo", "jobs", |_| async { - Ok::<_, redis::RedisError>(HashMap::new()) - }) + async fn resolve_consumer_treats_missing_or_invalid_identity_as_absent() { + for authoritative in [ + HashMap::new(), + str_map(&[("worker", "worker"), ("version", "invalid")]), + ] { + let queues = QueueState::default(); + let stream_key = queue_stream_key("demo", "jobs"); + let stale = + hydrate_consumer("demo", "jobs", &valid_consumer_hash("old-worker", "v1")).unwrap(); + queues + .registry + .write() + .await + .insert(stream_key.clone(), stale); + refresh_consumer_streams_for(&queues).await; + + let consumer = resolve_consumer_with_hash_loader( + &queues, + &stream_key, + "demo", + "jobs", + |_| async { Ok::<_, redis::RedisError>(authoritative) }, + ) .await .unwrap(); - assert_eq!(consumer, None); - assert!(!queues.registry.read().await.contains_key(&stream_key)); - assert!(queues.consumer_streams.read().await.is_empty()); + assert_eq!(consumer, None); + assert!(!queues.registry.read().await.contains_key(&stream_key)); + assert!(queues.consumer_streams.read().await.is_empty()); + } } } diff --git a/rust/scheduler/src/runtime_client.rs b/rust/scheduler/src/runtime_client.rs index 1df6716..5f313f1 100644 --- a/rust/scheduler/src/runtime_client.rs +++ b/rust/scheduler/src/runtime_client.rs @@ -5,6 +5,8 @@ use serde_json::Value as JsonValue; use crate::{AppState, Config}; use wdl_rust_common::internal_auth::INTERNAL_AUTH_HEADER; +const MAX_RUNTIME_RESPONSE_BYTES: usize = 1024 * 1024; + pub(crate) struct RuntimeResponse { pub(crate) status: Option, pub(crate) json: Option, @@ -70,7 +72,17 @@ pub(crate) async fn post_runtime( }; }; let status = response.status().as_u16(); - let text = response.text().await.unwrap_or_default(); + let text = match read_runtime_response(response).await { + Ok(text) => text, + Err(error) => { + return RuntimeResponse { + status: Some(status), + json: None, + text: None, + error: Some(error), + }; + } + }; let parsed = serde_json::from_str::(&text).ok(); RuntimeResponse { status: Some(status), @@ -80,6 +92,32 @@ pub(crate) async fn post_runtime( } } +async fn read_runtime_response(mut response: reqwest::Response) -> Result { + if response + .content_length() + .is_some_and(|length| length > MAX_RUNTIME_RESPONSE_BYTES as u64) + { + return Err("runtime response body exceeds 1048576 bytes".to_string()); + } + let mut body = Vec::new(); + while let Some(chunk) = response + .chunk() + .await + .map_err(|err| format!("failed to read runtime response body: {err}"))? + { + append_runtime_response_chunk(&mut body, &chunk)?; + } + String::from_utf8(body).map_err(|_| "runtime response body is not valid UTF-8".to_string()) +} + +fn append_runtime_response_chunk(body: &mut Vec, chunk: &[u8]) -> Result<(), String> { + if body.len().saturating_add(chunk.len()) > MAX_RUNTIME_RESPONSE_BYTES { + return Err("runtime response body exceeds 1048576 bytes".to_string()); + } + body.extend_from_slice(chunk); + Ok(()) +} + #[cfg(test)] mod tests { use super::*; @@ -115,4 +153,16 @@ mod tests { "error" ); } + + #[test] + fn runtime_response_body_accepts_the_limit_and_rejects_the_next_byte() { + let mut body = Vec::new(); + let exact_limit = vec![b'a'; MAX_RUNTIME_RESPONSE_BYTES]; + append_runtime_response_chunk(&mut body, &exact_limit).unwrap(); + assert_eq!(body.len(), MAX_RUNTIME_RESPONSE_BYTES); + assert_eq!( + append_runtime_response_chunk(&mut body, b"x"), + Err("runtime response body exceeds 1048576 bytes".to_string()) + ); + } } diff --git a/rust/scheduler/src/test_fixtures.rs b/rust/scheduler/src/test_fixtures.rs new file mode 100644 index 0000000..37fb0e9 --- /dev/null +++ b/rust/scheduler/src/test_fixtures.rs @@ -0,0 +1,77 @@ +use std::collections::HashMap; + +use serde::Deserialize; + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct SchedulerProjectionContract { + pub(crate) cron: CronProjectionContract, + pub(crate) queue_consumer: QueueConsumerProjectionContract, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct CronProjectionContract { + pub(crate) worker_index_key: String, + pub(crate) ns: String, + pub(crate) worker: String, + pub(crate) worker_key: String, + pub(crate) slot_ms: i64, + pub(crate) slot_key: String, + pub(crate) slot_expire_at: i64, + pub(crate) cron_id: String, + pub(crate) r#gen: i64, + pub(crate) reference: String, + pub(crate) meta: CronMetaContract, + pub(crate) entry: CronEntryContract, +} + +#[derive(Deserialize)] +pub(crate) struct CronMetaContract { + pub(crate) version: String, + pub(crate) json: String, +} + +#[derive(Deserialize)] +pub(crate) struct CronEntryContract { + pub(crate) cron: String, + pub(crate) timezone: String, + pub(crate) r#gen: i64, + pub(crate) json: String, +} + +#[derive(Deserialize)] +pub(crate) struct QueueConsumerProjectionContract { + pub(crate) ns: String, + pub(crate) queue: String, + pub(crate) worker: String, + pub(crate) version: String, + pub(crate) fields: HashMap, +} + +pub(crate) fn scheduler_projection_contract() -> SchedulerProjectionContract { + serde_json::from_str(include_str!( + "../../../tests/fixtures/scheduler-projection-contract.json" + )) + .expect("scheduler projection contract fixture must parse") +} + +pub(crate) fn version_tag_cases() -> Vec<(String, bool)> { + let fixture: serde_json::Value = + serde_json::from_str(include_str!("../../../tests/fixtures/version-tags.json")) + .expect("version tag fixture must parse"); + fixture["cases"] + .as_array() + .expect("version tag fixture cases must be an array") + .iter() + .map(|entry| { + ( + entry["tag"] + .as_str() + .expect("version tag must be a string") + .to_string(), + entry["parsed"].as_u64().is_some(), + ) + }) + .collect() +} diff --git a/rust/supervisor/Cargo.toml b/rust/supervisor/Cargo.toml index 46c6b81..cae5d0c 100644 --- a/rust/supervisor/Cargo.toml +++ b/rust/supervisor/Cargo.toml @@ -31,3 +31,6 @@ serde_json.workspace = true # Supervisor bins use #[tokio::main(flavor = "current_thread")]. tokio = { workspace = true, features = ["macros", "process", "rt", "signal", "sync", "time"] } wdl-rust-common = { workspace = true, default-features = false } + +[dev-dependencies] +wdl-rust-common = { workspace = true, default-features = false, features = ["test-support"] } diff --git a/rust/supervisor/src/config.rs b/rust/supervisor/src/config.rs index a7e2b8e..35e93e8 100644 --- a/rust/supervisor/src/config.rs +++ b/rust/supervisor/src/config.rs @@ -258,71 +258,43 @@ pub(crate) fn pick_do_compiled_config() -> &'static str { #[cfg(test)] mod tests { - use std::sync::{Mutex, OnceLock}; - use super::*; - - static ENV_LOCK: OnceLock> = OnceLock::new(); - - fn temp_env(key: &str, value: Option<&str>, f: impl FnOnce() -> R) -> R { - let _guard = ENV_LOCK - .get_or_init(|| Mutex::new(())) - .lock() - .expect("env mutex poisoned"); - let prev = std::env::var(key).ok(); - // SAFETY: these tests serialize environment mutation with ENV_LOCK, so no - // concurrent test can read or write the same process environment while this - // temporary override is active. - match value { - Some(v) => unsafe { std::env::set_var(key, v) }, - // SAFETY: Same serialization rationale as set_var above. - None => unsafe { std::env::remove_var(key) }, - } - let result = f(); - // SAFETY: ENV_LOCK is still held, so restoring the process environment is - // serialized with all other test environment mutation in this module. - match prev { - Some(p) => unsafe { std::env::set_var(key, p) }, - // SAFETY: Same serialization rationale as set_var above. - None => unsafe { std::env::remove_var(key) }, - } - result - } + use wdl_rust_common::test_env::with_temp_env; #[test] fn positive_int_env_accepts_only_strict_positive_integer_strings() { - temp_env("WDLTEST_VAL", None, || { + with_temp_env("WDLTEST_VAL", None, || { assert_eq!(positive_int_env("WDLTEST_", "VAL", 7), 7); }); - temp_env("WDLTEST_VAL", Some(""), || { + with_temp_env("WDLTEST_VAL", Some(""), || { assert_eq!(positive_int_env("WDLTEST_", "VAL", 7), 7); }); - temp_env("WDLTEST_VAL", Some("0"), || { + with_temp_env("WDLTEST_VAL", Some("0"), || { assert_eq!(positive_int_env("WDLTEST_", "VAL", 7), 7); }); - temp_env("WDLTEST_VAL", Some("-3"), || { + with_temp_env("WDLTEST_VAL", Some("-3"), || { assert_eq!(positive_int_env("WDLTEST_", "VAL", 7), 7); }); - temp_env("WDLTEST_VAL", Some("nope"), || { + with_temp_env("WDLTEST_VAL", Some("nope"), || { assert_eq!(positive_int_env("WDLTEST_", "VAL", 7), 7); }); - temp_env("WDLTEST_VAL", Some("12.9"), || { + with_temp_env("WDLTEST_VAL", Some("12.9"), || { assert_eq!(positive_int_env("WDLTEST_", "VAL", 7), 7); }); - temp_env("WDLTEST_VAL", Some("1e3"), || { + with_temp_env("WDLTEST_VAL", Some("1e3"), || { assert_eq!(positive_int_env("WDLTEST_", "VAL", 7), 7); }); - temp_env("WDLTEST_VAL", Some("18446744073709551616"), || { + with_temp_env("WDLTEST_VAL", Some("18446744073709551616"), || { assert_eq!(positive_int_env("WDLTEST_", "VAL", 7), 7); }); - temp_env("WDLTEST_VAL", Some("42"), || { + with_temp_env("WDLTEST_VAL", Some("42"), || { assert_eq!(positive_int_env("WDLTEST_", "VAL", 7), 42); }); } #[test] fn owner_ttl_ms_saturates_extreme_env_values() { - temp_env( + with_temp_env( "WDLTEST_OWNER_TTL_SECONDS", Some("18446744073709552"), || { diff --git a/rust/supervisor/src/drain.rs b/rust/supervisor/src/drain.rs index 8c7535f..1d649d4 100644 --- a/rust/supervisor/src/drain.rs +++ b/rust/supervisor/src/drain.rs @@ -126,39 +126,12 @@ pub(crate) fn build_request_id(prefix: &str) -> String { #[cfg(test)] mod tests { - use std::sync::{Mutex, OnceLock}; - use super::*; - - static ENV_LOCK: OnceLock> = OnceLock::new(); - - fn temp_env(key: &str, value: Option<&str>, f: impl FnOnce() -> R) -> R { - let _guard = ENV_LOCK - .get_or_init(|| Mutex::new(())) - .lock() - .expect("env mutex poisoned"); - let prev = std::env::var(key).ok(); - // SAFETY: tests hold ENV_LOCK for the full override/restore window, so process - // environment mutation is serialized. - match value { - Some(v) => unsafe { std::env::set_var(key, v) }, - // SAFETY: Same serialization rationale as set_var above. - None => unsafe { std::env::remove_var(key) }, - } - let result = f(); - // SAFETY: ENV_LOCK is still held, so restoring the environment cannot race - // with another test in this module. - match prev { - Some(p) => unsafe { std::env::set_var(key, p) }, - // SAFETY: Same serialization rationale as set_var above. - None => unsafe { std::env::remove_var(key) }, - } - result - } + use wdl_rust_common::test_env::with_temp_env; #[test] fn build_request_id_starts_with_prefix() { - temp_env("HOSTNAME", Some("test-host"), || { + with_temp_env("HOSTNAME", Some("test-host"), || { let id = build_request_id("do-drain"); assert!( id.starts_with("do-drain-test-host-"), diff --git a/rust/supervisor/src/renew.rs b/rust/supervisor/src/renew.rs index 2554d70..652e5b0 100644 --- a/rust/supervisor/src/renew.rs +++ b/rust/supervisor/src/renew.rs @@ -156,51 +156,12 @@ fn log_renew_error( #[cfg(test)] mod tests { - use std::sync::{Mutex, OnceLock}; - use super::*; - - static ENV_LOCK: OnceLock> = OnceLock::new(); - - struct EnvRestore { - key: String, - prev: Option, - } - - impl Drop for EnvRestore { - fn drop(&mut self) { - // SAFETY: tests hold ENV_LOCK for the full override/restore window, so - // restoring the process environment is serialized. - match &self.prev { - Some(prev) => unsafe { std::env::set_var(&self.key, prev) }, - None => unsafe { std::env::remove_var(&self.key) }, - } - } - } - - fn temp_env(key: &str, value: Option<&str>, f: impl FnOnce() -> R) -> R { - let _guard = ENV_LOCK - .get_or_init(|| Mutex::new(())) - .lock() - .expect("env mutex poisoned"); - let prev = std::env::var(key).ok(); - let _restore = EnvRestore { - key: key.to_string(), - prev, - }; - // SAFETY: tests hold ENV_LOCK for the full override/restore window, so process - // environment mutation is serialized. - match value { - Some(v) => unsafe { std::env::set_var(key, v) }, - // SAFETY: Same serialization rationale as set_var above. - None => unsafe { std::env::remove_var(key) }, - } - f() - } + use wdl_rust_common::test_env::with_temp_env; #[test] fn start_fails_before_spawn_when_internal_auth_token_is_missing() { - temp_env("WDL_INTERNAL_AUTH_TOKEN", None, || { + with_temp_env("WDL_INTERNAL_AUTH_TOKEN", None, || { let client = Arc::new(reqwest::Client::new()); match start(&crate::D1_CONFIG, client) { Ok(_) => panic!("start must reject missing token"), diff --git a/rust/workflows/Cargo.toml b/rust/workflows/Cargo.toml index b4c70f0..1053537 100644 --- a/rust/workflows/Cargo.toml +++ b/rust/workflows/Cargo.toml @@ -25,3 +25,6 @@ serde_json.workspace = true sha2.workspace = true tokio = { workspace = true, features = ["macros", "net", "rt-multi-thread", "signal", "sync", "time"] } wdl-rust-common = { workspace = true, features = ["axum"] } + +[dev-dependencies] +wdl-rust-common = { workspace = true, features = ["axum", "test-support"] } diff --git a/rust/workflows/src/api.rs b/rust/workflows/src/api.rs index ddb963d..8c8e247 100644 --- a/rust/workflows/src/api.rs +++ b/rust/workflows/src/api.rs @@ -16,6 +16,7 @@ mod limits; mod model; mod payload; mod pending_create; +mod pending_restart; mod progress; mod redis_script; mod retention; @@ -25,7 +26,7 @@ mod status; mod tick; use active_export::{ ensure_worker_not_deleting, request_with_active_version, verify_active_workflow_current, - verify_active_workflow_export, verify_workflow_def, + verify_workflow_def, }; pub(crate) use create::{create_batch, create_instance}; pub(crate) use do_alarms::{ @@ -55,7 +56,7 @@ use limits::{ MAX_WORKFLOW_EVENT_TYPE_BYTES, MAX_WORKFLOW_INSTANCE_PAYLOAD_BYTES, MAX_WORKFLOW_JSON_BODY_BYTES, MAX_WORKFLOW_PARAMS_BYTES, MAX_WORKFLOW_RESULT_BYTES, MAX_WORKFLOW_RUNTIME_RESPONSE_BYTES, MAX_WORKFLOW_STEP_CONFIG_BYTES, - MAX_WORKFLOW_STEP_NAME_BYTES, READY_SHARDS, + MAX_WORKFLOW_STEP_NAME_BYTES, READY_SHARDS, WORKFLOW_PAYLOAD_TOO_LARGE_CODE, }; pub(crate) use model::{ CreateBatchResponse, EventRecord, InstanceIdentity, InstanceResponse, LifecycleBlocker, @@ -72,6 +73,10 @@ use pending_create::{ pending_create_cleanup_from_state, pending_create_expired, pending_create_token, public_state_or_empty, wait_for_public_create_state, }; +use pending_restart::{ + active_pending_restart_blockers, create_pending_restart, pending_restart_marker, + remove_pending_restart, renew_pending_restart, +}; use progress::{ spawn_progress_from_identity, spawn_progress_from_request, spawn_progress_from_step, }; @@ -92,6 +97,18 @@ use status::{ }; pub(crate) use tick::tick_workflows; +fn runtime_endpoint(app: &AppState, ns: &str, path: &str) -> String { + let (host, port) = if ns == "__system__" { + ( + &app.config.system_runtime_host, + app.config.system_runtime_port, + ) + } else { + (&app.config.runtime_host, app.config.runtime_port) + }; + format!("http://{host}:{port}{path}") +} + #[cfg(test)] pub(crate) use execution::{ COMMIT_STEP_ERROR_SCRIPT, COMMIT_STEP_RECORD_SCRIPT, COMMIT_STEP_SUCCESS_SCRIPT, diff --git a/rust/workflows/src/api/active_export.rs b/rust/workflows/src/api/active_export.rs index 0236343..7da1295 100644 --- a/rust/workflows/src/api/active_export.rs +++ b/rust/workflows/src/api/active_export.rs @@ -1,5 +1,5 @@ use serde_json::Value as JsonValue; -use wdl_rust_common::version::routes_key; +use wdl_rust_common::version::{routes_key, worker_delete_lock_key}; use crate::{AppState, WorkflowError, WorkflowResult}; @@ -124,47 +124,6 @@ pub(super) async fn verify_workflow_def_values( Ok(()) } -pub(super) async fn verify_active_workflow_export( - state: &AppState, - req: &WorkflowRequest, -) -> WorkflowResult<()> { - let key = bundle_key(&req.ns, &req.worker, &req.frozen_version)?; - let raw: Option = state - .control_redis - .with_conn(async |mut conn| { - redis::cmd("HGET") - .arg(key) - .arg("__meta__") - .query_async(&mut conn) - .await - }) - .await?; - let raw = raw.ok_or_else(|| { - WorkflowError::not_exported("Workflow is not exported by the active worker version") - })?; - let meta: JsonValue = serde_json::from_str(&raw).map_err(|err| { - WorkflowError::invalid_state(format!("Active worker metadata is corrupt: {err}")) - })?; - let workflows = meta - .get("workflows") - .and_then(JsonValue::as_array) - .ok_or_else(|| { - WorkflowError::not_exported("Workflow is not exported by the active worker version") - })?; - let exported = workflows.iter().any(|entry| { - entry.get("name").and_then(JsonValue::as_str) == Some(req.workflow_name.as_str()) - && entry.get("workflowKey").and_then(JsonValue::as_str) - == Some(req.workflow_key.as_str()) - && entry.get("className").and_then(JsonValue::as_str) == Some(req.class_name.as_str()) - }); - if !exported { - return Err(WorkflowError::not_exported( - "Workflow is not exported by the active worker version", - )); - } - Ok(()) -} - pub(super) async fn verify_active_workflow_current( state: &AppState, req: &WorkflowRequest, @@ -179,7 +138,7 @@ pub(super) async fn verify_active_workflow_current( .await?; if active.version != req.frozen_version || active.class_name != req.class_name { return Err(WorkflowError::not_exported( - "Workflow active worker export changed during create", + "Workflow active worker export changed during mutation", )); } Ok(()) @@ -190,7 +149,7 @@ pub(super) async fn ensure_worker_not_deleting( ns: &str, worker: &str, ) -> WorkflowResult<()> { - let lock_key = format!("worker-delete-lock:{ns}:{worker}"); + let lock_key = worker_delete_lock_key(ns, worker); let lock_exists: i64 = state .control_redis .with_conn(async |mut conn| { @@ -202,7 +161,7 @@ pub(super) async fn ensure_worker_not_deleting( .await?; if lock_exists > 0 { return Err(WorkflowError::deleting( - "Worker is being deleted; workflow creation is blocked", + "Worker is being deleted; workflow mutation is blocked", )); } Ok(()) diff --git a/rust/workflows/src/api/create.rs b/rust/workflows/src/api/create.rs index 1aa05b2..b574dc5 100644 --- a/rust/workflows/src/api/create.rs +++ b/rust/workflows/src/api/create.rs @@ -240,9 +240,9 @@ pub(crate) async fn create_batch( )); } if req.entries.len() > MAX_CREATE_BATCH_SIZE { - return Err(WorkflowError::request_too_large( - "createBatch exceeds 100 instance limit", - )); + return Err(WorkflowError::request_too_large(format!( + "createBatch exceeds {MAX_CREATE_BATCH_SIZE} instance limit" + ))); } let req = request_with_active_version(state, req).await?; validate_identity(&req)?; diff --git a/rust/workflows/src/api/do_alarms/dispatch.rs b/rust/workflows/src/api/do_alarms/dispatch.rs index 887defe..fd4a908 100644 --- a/rust/workflows/src/api/do_alarms/dispatch.rs +++ b/rust/workflows/src/api/do_alarms/dispatch.rs @@ -4,7 +4,9 @@ use serde::Serialize; use serde_json::{Value as JsonValue, json}; use wdl_rust_common::internal_auth::INTERNAL_AUTH_HEADER; use wdl_rust_common::time::now_ms; -use wdl_rust_common::version::{do_storage_id_key, routes_key, worker_versions_key}; +use wdl_rust_common::version::{ + do_storage_id_key, parse_version_tag, routes_key, worker_versions_key, +}; use crate::{ AppState, DoAlarmJobKeys, LogLevel, WorkflowError, WorkflowResult, do_alarm_shard_queue_keys, @@ -241,6 +243,8 @@ async fn resolve_alarm_dispatch_version( } if let Some(active_version) = active_version { + parse_version_tag(&active_version) + .map_err(|_| WorkflowError::invalid_state("Active worker version is invalid"))?; log( app, LogLevel::Info, diff --git a/rust/workflows/src/api/do_alarms/model.rs b/rust/workflows/src/api/do_alarms/model.rs index 723aeb1..df180ce 100644 --- a/rust/workflows/src/api/do_alarms/model.rs +++ b/rust/workflows/src/api/do_alarms/model.rs @@ -1,6 +1,9 @@ use std::collections::HashMap; use serde::{Deserialize, Serialize}; +use wdl_rust_common::hash::fnv1a32; +use wdl_rust_common::identity::{is_valid_runtime_load_ns, is_valid_worker_name}; +use wdl_rust_common::version::parse_version_tag; use crate::{ DoAlarmJobKeys, WorkflowError, WorkflowResult, do_alarm_by_worker_key, do_alarm_job_id, @@ -83,27 +86,19 @@ pub(crate) struct DoAlarmTickCounters { pub(crate) skipped: usize, } -pub(super) fn validate_non_empty(value: &str, field: &'static str) -> WorkflowResult<()> { - if value.is_empty() { - return Err(WorkflowError::invalid_request(format!( - "{field} is required" - ))); - } - Ok(()) -} - pub(super) fn validate_set_request(req: &DoAlarmSetRequest) -> WorkflowResult<()> { - // These identity fields arrive from the private do-runtime alarm shim. They are - // hashed into job ids before touching Redis key names, so non-empty validation - // preserves the internal contract without imposing tenant namespace/name grammar - // on Durable Object class names or object names. - validate_non_empty(&req.ns, "ns")?; - validate_non_empty(&req.worker, "worker")?; - validate_non_empty(&req.version, "version")?; - validate_non_empty(&req.do_storage_id, "doStorageId")?; - validate_non_empty(&req.class_name, "className")?; - validate_non_empty(&req.object_name, "objectName")?; - validate_non_empty(&req.token, "token")?; + // The job id is hashed, but the by-worker index embeds ns/worker directly. + // Validate the complete dispatch identity before either key family is built. + validate_protocol_request_field(&req.ns, "ns", is_valid_runtime_load_ns)?; + validate_protocol_request_field(&req.worker, "worker", is_valid_worker_name)?; + validate_protocol_request_field(&req.version, "version", |value| { + parse_version_tag(value).is_ok() + })?; + validate_protocol_request_field(&req.do_storage_id, "doStorageId", is_storage_id)?; + validate_protocol_request_field(&req.class_name, "className", is_class_name)?; + validate_opaque_request_field(&req.object_name, "objectName")?; + validate_opaque_request_field(&req.token, "token")?; + validate_alarm_host_id(&req.do_storage_id, &req.class_name, &req.object_name)?; if req.scheduled_time <= 0 { return Err(WorkflowError::invalid_request( "scheduledTime must be a positive Unix millisecond timestamp", @@ -113,12 +108,20 @@ pub(super) fn validate_set_request(req: &DoAlarmSetRequest) -> WorkflowResult<() } pub(super) fn validate_delete_request(req: &DoAlarmDeleteRequest) -> WorkflowResult<()> { - validate_non_empty(&req.ns, "ns")?; - validate_non_empty(&req.worker, "worker")?; - validate_non_empty(&req.do_storage_id, "doStorageId")?; - validate_non_empty(&req.class_name, "className")?; - validate_non_empty(&req.object_name, "objectName")?; - validate_non_empty(&req.token, "token")?; + validate_protocol_request_field(&req.ns, "ns", is_valid_runtime_load_ns)?; + validate_protocol_request_field(&req.worker, "worker", is_valid_worker_name)?; + validate_protocol_request_field(&req.do_storage_id, "doStorageId", is_storage_id)?; + validate_protocol_request_field(&req.class_name, "className", is_class_name)?; + validate_opaque_request_field(&req.object_name, "objectName")?; + validate_opaque_request_field(&req.token, "token")?; + validate_alarm_host_id(&req.do_storage_id, &req.class_name, &req.object_name)?; + Ok(()) +} + +pub(super) fn validate_cleanup_request(req: &DoAlarmCleanupRequest) -> WorkflowResult<()> { + validate_protocol_request_field(&req.ns, "ns", is_valid_runtime_load_ns)?; + validate_protocol_request_field(&req.worker, "worker", is_valid_worker_name)?; + validate_protocol_request_field(&req.do_storage_id, "doStorageId", is_storage_id)?; Ok(()) } @@ -147,6 +150,23 @@ pub(super) fn map_hgetall(flat: Vec) -> HashMap { } const MAX_PROTOCOL_STRING_BYTES: usize = 512; +const DO_HOST_SHARD_COUNT: u32 = 16; + +fn alarm_host_id_len(do_storage_id: &str, class_name: &str, object_name: &str) -> usize { + let shard = fnv1a32(object_name.as_bytes()) % DO_HOST_SHARD_COUNT; + do_storage_id.len() + class_name.len() + ":".len() + ":shard".len() + shard.to_string().len() +} + +fn validate_alarm_host_id( + do_storage_id: &str, + class_name: &str, + object_name: &str, +) -> WorkflowResult<()> { + if alarm_host_id_len(do_storage_id, class_name, object_name) > MAX_PROTOCOL_STRING_BYTES { + return Err(WorkflowError::invalid_request("hostId is invalid")); + } + Ok(()) +} fn required_field<'a>( state: &'a HashMap, @@ -167,6 +187,30 @@ fn has_control_char(value: &str) -> bool { value.bytes().any(|byte| byte < 0x20 || byte == 0x7f) } +fn is_protocol_string(value: &str, valid: impl Fn(&str) -> bool) -> bool { + !value.is_empty() + && value.len() <= MAX_PROTOCOL_STRING_BYTES + && !has_control_char(value) + && valid(value) +} + +fn validate_protocol_request_field( + value: &str, + field: &'static str, + valid: impl Fn(&str) -> bool, +) -> WorkflowResult<()> { + if !is_protocol_string(value, valid) { + return Err(WorkflowError::invalid_request(format!( + "{field} is invalid" + ))); + } + Ok(()) +} + +fn validate_opaque_request_field(value: &str, field: &'static str) -> WorkflowResult<()> { + validate_protocol_request_field(value, field, |_| true) +} + fn is_class_name(value: &str) -> bool { let mut chars = value.chars(); let Some(first) = chars.next() else { @@ -176,57 +220,30 @@ fn is_class_name(value: &str) -> bool { && chars.all(|ch| ch == '_' || ch == '$' || ch.is_ascii_alphanumeric()) } -fn is_worker_name(value: &str) -> bool { - let bytes = value.as_bytes(); - matches!(bytes.first(), Some(first) if first.is_ascii_alphanumeric()) - && bytes.len() <= 255 - && bytes - .iter() - .all(|byte| byte.is_ascii_alphanumeric() || *byte == b'_' || *byte == b'-') -} - -fn is_ns_field(value: &str) -> bool { - let bytes = value.as_bytes(); - if value.starts_with("__") && value.ends_with("__") && bytes.len() > 4 { - return bytes[2..bytes.len() - 2].iter().all(|byte| { - byte.is_ascii_lowercase() || byte.is_ascii_digit() || *byte == b'_' || *byte == b'-' - }); - } - bytes - .iter() - .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || *byte == b'-') -} - fn is_storage_id(value: &str) -> bool { value.bytes().all(|byte| { byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_' || byte == b'-' }) } -fn is_version(value: &str) -> bool { - value.strip_prefix('v').is_some_and(|suffix| { - !suffix.is_empty() && suffix.bytes().all(|byte| byte.is_ascii_digit()) - }) -} - fn protocol_string_field<'a>( state: &'a HashMap, field: &'static str, valid: impl Fn(&str) -> bool, ) -> WorkflowResult<&'a str> { let value = required_field(state, field)?; - if value.len() > MAX_PROTOCOL_STRING_BYTES || has_control_char(value) || !valid(value) { + if !is_protocol_string(value, valid) { return Err(invalid_job_field(field)); } Ok(value) } -fn object_name_field<'a>( +fn opaque_string_field<'a>( state: &'a HashMap, field: &'static str, ) -> WorkflowResult<&'a str> { let value = required_field(state, field)?; - if value.len() > MAX_PROTOCOL_STRING_BYTES || has_control_char(value) { + if !is_protocol_string(value, |_| true) { return Err(invalid_job_field(field)); } Ok(value) @@ -258,16 +275,25 @@ pub(super) fn job_from_state( state: HashMap, ) -> WorkflowResult { parse_positive_i64_field(&state, "dueAtMs")?; + let do_storage_id = protocol_string_field(&state, "doStorageId", is_storage_id)?; + let class_name = protocol_string_field(&state, "className", is_class_name)?; + let object_name = opaque_string_field(&state, "objectName")?; + if alarm_host_id_len(do_storage_id, class_name, object_name) > MAX_PROTOCOL_STRING_BYTES { + return Err(invalid_job_field("hostId")); + } Ok(DoAlarmJob { job_id, - ns: protocol_string_field(&state, "ns", is_ns_field)?.to_string(), - worker: protocol_string_field(&state, "worker", is_worker_name)?.to_string(), - version: protocol_string_field(&state, "scheduledVersion", is_version)?.to_string(), - do_storage_id: protocol_string_field(&state, "doStorageId", is_storage_id)?.to_string(), - class_name: protocol_string_field(&state, "className", is_class_name)?.to_string(), - object_name: object_name_field(&state, "objectName")?.to_string(), - row_token: required_field(&state, "rowToken")?.to_string(), - run_token: required_field(&state, "runToken")?.to_string(), + ns: protocol_string_field(&state, "ns", is_valid_runtime_load_ns)?.to_string(), + worker: protocol_string_field(&state, "worker", is_valid_worker_name)?.to_string(), + version: protocol_string_field(&state, "scheduledVersion", |value| { + parse_version_tag(value).is_ok() + })? + .to_string(), + do_storage_id: do_storage_id.to_string(), + class_name: class_name.to_string(), + object_name: object_name.to_string(), + row_token: opaque_string_field(&state, "rowToken")?.to_string(), + run_token: opaque_string_field(&state, "runToken")?.to_string(), retry_count: parse_u64_field(&state, "retryCount")?, }) } @@ -276,6 +302,45 @@ pub(super) fn job_from_state( mod tests { use super::*; + fn valid_set_request() -> DoAlarmSetRequest { + DoAlarmSetRequest { + ns: "demo".to_string(), + worker: "alarms".to_string(), + version: "v1".to_string(), + do_storage_id: "do_abc".to_string(), + class_name: "AlarmCounter".to_string(), + object_name: "alpha".to_string(), + scheduled_time: 123_456_789, + retry_count: 0, + token: "row-token".to_string(), + } + } + + fn valid_delete_request() -> DoAlarmDeleteRequest { + DoAlarmDeleteRequest { + ns: "demo".to_string(), + worker: "alarms".to_string(), + do_storage_id: "do_abc".to_string(), + class_name: "AlarmCounter".to_string(), + object_name: "alpha".to_string(), + token: "row-token".to_string(), + } + } + + fn fixture_string(value: &serde_json::Value) -> String { + if let Some(value) = value.as_str() { + return value.to_string(); + } + value["repeat"] + .as_str() + .expect("fixture repeat is a string") + .repeat( + value["count"] + .as_u64() + .expect("fixture count is an integer") as usize, + ) + } + fn valid_job_state() -> HashMap { HashMap::from([ ("ns".to_string(), "demo".to_string()), @@ -291,6 +356,126 @@ mod tests { ]) } + #[test] + fn do_alarm_mutation_ingress_rejects_noncanonical_identity() { + for (field, value) in [ + ("ns", "Demo"), + ("worker", "bad:worker"), + ("version", "v01"), + ("doStorageId", "DO_ABC"), + ("className", "bad-name"), + ("objectName", "bad\nname"), + ("token", "bad\ntoken"), + ] { + let mut req = valid_set_request(); + match field { + "ns" => req.ns = value.to_string(), + "worker" => req.worker = value.to_string(), + "version" => req.version = value.to_string(), + "doStorageId" => req.do_storage_id = value.to_string(), + "className" => req.class_name = value.to_string(), + "objectName" => req.object_name = value.to_string(), + "token" => req.token = value.to_string(), + _ => unreachable!(), + } + let err = validate_set_request(&req).expect_err("invalid set request"); + assert_eq!(err.code, "invalid_request"); + assert!(err.message.contains(field), "{}", err.message); + } + + let delete = DoAlarmDeleteRequest { + ns: "bad:ns".to_string(), + worker: "alarms".to_string(), + do_storage_id: "do_abc".to_string(), + class_name: "AlarmCounter".to_string(), + object_name: "alpha".to_string(), + token: "row-token".to_string(), + }; + assert!(validate_delete_request(&delete).is_err()); + + let cleanup = DoAlarmCleanupRequest { + ns: "demo".to_string(), + worker: "alarms".to_string(), + do_storage_id: "bad/storage".to_string(), + }; + assert!(validate_cleanup_request(&cleanup).is_err()); + } + + #[test] + fn do_alarm_protocol_fields_match_cross_language_fixture() { + let fixture: serde_json::Value = serde_json::from_str(include_str!( + "../../../../../tests/fixtures/do-alarm-identity.json" + )) + .expect("DO alarm identity fixture parses"); + assert_eq!( + fixture["maxBytes"].as_u64(), + Some(MAX_PROTOCOL_STRING_BYTES as u64) + ); + assert_eq!( + fixture["hostShardCount"].as_u64(), + Some(DO_HOST_SHARD_COUNT as u64) + ); + for case in fixture["cases"] + .as_array() + .expect("DO alarm identity fixture cases is an array") + { + let do_storage_id = fixture_string(&case["doStorageId"]); + let class_name = fixture_string(&case["className"]); + let object_name = fixture_string(&case["objectName"]); + let token = fixture_string(&case["token"]); + let mut set = valid_set_request(); + set.do_storage_id = do_storage_id.clone(); + set.class_name = class_name.clone(); + set.object_name = object_name.clone(); + set.token = token.clone(); + let mut delete = valid_delete_request(); + delete.do_storage_id = do_storage_id; + delete.class_name = class_name; + delete.object_name = object_name; + delete.token = token; + let valid = case["valid"].as_bool().expect("valid is a boolean"); + assert_eq!( + validate_set_request(&set).is_ok(), + valid, + "set:{}", + case["name"].as_str().expect("name is a string") + ); + assert_eq!( + validate_delete_request(&delete).is_ok(), + valid, + "delete:{}", + case["name"].as_str().expect("name is a string") + ); + let mut state = valid_job_state(); + state.insert("doStorageId".to_string(), set.do_storage_id.clone()); + state.insert("className".to_string(), set.class_name.clone()); + state.insert("objectName".to_string(), set.object_name.clone()); + state.insert("rowToken".to_string(), set.token.clone()); + assert_eq!( + job_from_state("job".to_string(), state).is_ok(), + valid, + "persisted:{}", + case["name"].as_str().expect("name is a string") + ); + } + } + + #[test] + fn do_alarm_json_rejects_unpaired_surrogate_object_names() { + let request = r#"{ + "ns":"demo", + "worker":"alarms", + "version":"v1", + "doStorageId":"do_abc", + "className":"AlarmCounter", + "objectName":"\ud800", + "scheduledTime":123456789, + "retryCount":0, + "token":"row-token" + }"#; + assert!(serde_json::from_str::(request).is_err()); + } + #[test] fn do_alarm_job_state_requires_valid_due_time() { for due_at_ms in [None, Some(""), Some("bad"), Some("0"), Some("-1")] { @@ -316,10 +501,15 @@ mod tests { #[test] fn do_alarm_job_state_rejects_malformed_dispatch_identity() { for (field, value) in [ + ("ns", ""), ("ns", "Demo"), + ("ns", "admin"), + ("ns", "__community__"), ("ns", "____"), ("worker", "-bad"), ("scheduledVersion", "1"), + ("scheduledVersion", "v0"), + ("scheduledVersion", "v01"), ("doStorageId", "DO_ABC"), ("className", "bad-name"), ("objectName", "bad\nname"), @@ -338,4 +528,25 @@ mod tests { ); } } + + #[test] + fn do_alarm_set_version_matches_cross_language_fixture() { + let fixture: serde_json::Value = serde_json::from_str(include_str!( + "../../../../../tests/fixtures/version-tags.json" + )) + .expect("version tag fixture parses"); + for case in fixture["cases"] + .as_array() + .expect("version tag fixture cases is an array") + { + let tag = case["tag"].as_str().expect("version tag is a string"); + let mut req = valid_set_request(); + req.version = tag.to_string(); + assert_eq!( + validate_set_request(&req).is_ok(), + case["parsed"].as_u64().is_some(), + "{tag:?}" + ); + } + } } diff --git a/rust/workflows/src/api/do_alarms/mutations.rs b/rust/workflows/src/api/do_alarms/mutations.rs index 2509541..56af2d8 100644 --- a/rust/workflows/src/api/do_alarms/mutations.rs +++ b/rust/workflows/src/api/do_alarms/mutations.rs @@ -8,7 +8,7 @@ use crate::{AppState, LogLevel, WorkflowResult, do_alarm_by_worker_key, log}; use super::super::eval_script; use super::model::{ DoAlarmCleanupRequest, DoAlarmDeleteRequest, DoAlarmMutationResponse, DoAlarmSetRequest, - job_keys_for_identity, validate_delete_request, validate_non_empty, validate_set_request, + job_keys_for_identity, validate_cleanup_request, validate_delete_request, validate_set_request, }; use super::scripts::{ CLEANUP_DO_ALARM_FOR_STORAGE_SCRIPT, DELETE_DO_ALARM_SCRIPT, SET_DO_ALARM_SCRIPT, @@ -177,9 +177,7 @@ pub(crate) async fn cleanup_do_alarms_for_worker( app: &AppState, req: DoAlarmCleanupRequest, ) -> WorkflowResult { - validate_non_empty(&req.ns, "ns")?; - validate_non_empty(&req.worker, "worker")?; - validate_non_empty(&req.do_storage_id, "doStorageId")?; + validate_cleanup_request(&req)?; let by_worker = do_alarm_by_worker_key(&req.ns, &req.worker); let snapshot_key = format!("{by_worker}:cleanup-snapshot:{}", random_hex_64()); let mut cursor = 0_u64; diff --git a/rust/workflows/src/api/lifecycle/cleanup.rs b/rust/workflows/src/api/lifecycle/cleanup.rs index f0cc240..db78df2 100644 --- a/rust/workflows/src/api/lifecycle/cleanup.rs +++ b/rust/workflows/src/api/lifecycle/cleanup.rs @@ -4,9 +4,9 @@ use crate::{AppState, WorkflowResult, by_version_key, by_worker_key}; use super::super::{ LIFECYCLE_BLOCKER_LIMIT, LifecycleBlocker, LifecycleCheckRequest, LifecycleCheckResponse, - PendingCreateCleanup, cleanup_pending_create_identity, is_pending_create, - parse_workflow_referrer_member, pending_create_cleanup_from_state, pending_create_expired, - read_state_by_id, require_non_empty, + PendingCreateCleanup, active_pending_restart_blockers, cleanup_pending_create_identity, + is_pending_create, parse_workflow_referrer_member, pending_create_cleanup_from_state, + pending_create_expired, read_state_by_id, require_non_empty, }; enum LifecycleMemberState { @@ -55,8 +55,22 @@ pub(crate) async fn check_delete_lifecycle( Some(version) => by_version_key(&req.ns, &req.worker, version), None => by_worker_key(&req.ns, &req.worker), }; + let (pending_count, mut blockers) = match &req.version { + Some(version) => { + active_pending_restart_blockers( + state, + &req.ns, + &req.worker, + version, + req.allow_cleanup, + LIFECYCLE_BLOCKER_LIMIT, + ) + .await? + } + None => (0, Vec::new()), + }; let count_key = key.clone(); - let count: usize = state + let referrer_count: usize = state .redis .with_conn(async |mut conn| { redis::cmd("SCARD") @@ -65,6 +79,7 @@ pub(crate) async fn check_delete_lifecycle( .await }) .await?; + let count = referrer_count.saturating_add(pending_count); if count == 0 { return Ok(LifecycleCheckResponse { allowed: true, @@ -72,8 +87,14 @@ pub(crate) async fn check_delete_lifecycle( blockers: Vec::new(), }); } + if blockers.len() >= LIFECYCLE_BLOCKER_LIMIT { + return Ok(LifecycleCheckResponse { + allowed: false, + count, + blockers, + }); + } let mut cursor = 0_u64; - let mut blockers = Vec::new(); let mut expired_pending = Vec::new(); loop { let scan_key = key.clone(); diff --git a/rust/workflows/src/api/lifecycle/restart.rs b/rust/workflows/src/api/lifecycle/restart.rs index 841f5cb..16845eb 100644 --- a/rust/workflows/src/api/lifecycle/restart.rs +++ b/rust/workflows/src/api/lifecycle/restart.rs @@ -6,16 +6,23 @@ use crate::{ }; use super::super::{ - InstanceResponse, InstanceRouteKeys, WorkflowRequest, eval_script, identity_from_state, - instance_id, payload_bytes_arg, read_public_state, request_with_active_version, - validate_identity, verify_active_workflow_export, verify_workflow_def, - workflow_referrer_member, + InstanceResponse, InstanceRouteKeys, WorkflowRequest, create_pending_restart, + ensure_worker_not_deleting, eval_script, identity_from_state, instance_id, payload_bytes_arg, + pending_restart_marker, read_public_state, remove_pending_restart, renew_pending_restart, + request_with_active_version, validate_identity, verify_active_workflow_current, + verify_workflow_def, workflow_referrer_member, }; use super::common::{ TransitionSignal, next_generation, stale_transition_response, successful_transition_response, }; const RESTART_SCRIPT: &str = r#" +local marker_score = redis.call("ZSCORE", KEYS[16], ARGV[12]) +local marker_time = redis.call("TIME") +local marker_now = tonumber(marker_time[1]) * 1000 + math.floor(tonumber(marker_time[2]) / 1000) +if not marker_score or tonumber(marker_score) <= marker_now then + return -1 +end local status = redis.call("HGET", KEYS[1], "status") if not status then return 0 @@ -36,6 +43,7 @@ redis.call("HDEL", KEYS[1], "outputRef", "errorRef", "errorCode", "errorMessage" redis.call("HDEL", KEYS[1], "runToken", "runLeaseExpiresAtMs", "waitingEventIndexPrefix") redis.call("SADD", KEYS[7], ARGV[6]) redis.call("SADD", KEYS[13], ARGV[10]) +redis.call("ZREM", KEYS[16], ARGV[12]) redis.call("ZREM", KEYS[8], ARGV[6]) redis.call("SREM", KEYS[9], ARGV[7]) redis.call("SADD", KEYS[10], ARGV[7]) @@ -45,14 +53,14 @@ redis.call("ZADD", KEYS[14], ARGV[3], ARGV[11]) return 1 "#; +const RESTART_LEASE_EXPIRED: i64 = -1; + pub(crate) async fn restart_instance( state: &AppState, req: WorkflowRequest, ) -> WorkflowResult { let req = request_with_active_version(state, req).await?; validate_identity(&req)?; - verify_active_workflow_export(state, &req).await?; - verify_workflow_def(state, &req).await?; let id = instance_id(&req)?.to_string(); let mut existing = read_public_state(state, &req).await?; if existing.is_empty() { @@ -84,6 +92,28 @@ pub(crate) async fn restart_instance( }) .await? .ok_or_else(|| WorkflowError::payload_missing("Workflow params payload is missing"))?; + ensure_worker_not_deleting(state, &req.ns, &req.worker).await?; + let pending_restart = pending_restart_marker(state, &req, &id); + create_pending_restart(state, &pending_restart).await?; + if let Err(err) = verify_active_workflow_current(state, &req).await { + remove_pending_restart(state, &pending_restart).await?; + return Err(err); + } + if let Err(err) = verify_workflow_def(state, &req).await { + remove_pending_restart(state, &pending_restart).await?; + return Err(err); + } + // Refresh immediately before the final delete-lock check. A version delete + // that starts on either side of this check sees the lock or this blocker. + if !renew_pending_restart(state, &pending_restart).await? { + return Err(WorkflowError::conflict( + "Workflow restart target lease expired; retry restart", + )); + } + if let Err(err) = ensure_worker_not_deleting(state, &req.ns, &req.worker).await { + remove_pending_restart(state, &pending_restart).await?; + return Err(err); + } let shard = keys.shard(); let ready = keys.ready(); let due = keys.due(); @@ -102,7 +132,7 @@ pub(crate) async fn restart_instance( let stored_payloads_key = payloads_key.clone(); let request_version = req.frozen_version.clone(); let params_bytes = payload_bytes_arg(¶ms_json); - let updated: i64 = eval_script( + let updated: i64 = match eval_script( state, RESTART_SCRIPT, &[ @@ -121,6 +151,7 @@ pub(crate) async fn restart_instance( ready_active_key(), &by_workflow, &event_index_key, + &pending_restart.key, ], &[ &expected_generation, @@ -134,10 +165,24 @@ pub(crate) async fn restart_instance( ¶ms_bytes, &shard_arg, &identity.instance_id, + &pending_restart.member, ], ) - .await?; + .await + { + Ok(updated) => updated, + Err(err) => { + remove_pending_restart(state, &pending_restart).await?; + return Err(err); + } + }; + if updated == RESTART_LEASE_EXPIRED { + return Err(WorkflowError::conflict( + "Workflow restart target lease expired; retry restart", + )); + } if updated != 1 { + remove_pending_restart(state, &pending_restart).await?; return stale_transition_response(state, &identity, &id).await; } existing.insert("status".to_string(), "queued".to_string()); @@ -189,10 +234,21 @@ mod tests { #[test] fn restart_updates_class_name_for_active_version() { + assert!( + RESTART_SCRIPT + .contains(r#"local marker_score = redis.call("ZSCORE", KEYS[16], ARGV[12])"#) + ); + assert!(RESTART_SCRIPT.contains(r#"local marker_time = redis.call("TIME")"#)); assert!(RESTART_SCRIPT.contains(r#""className", ARGV[5]"#)); assert!(RESTART_SCRIPT.contains(r#"redis.call("SADD", KEYS[7], ARGV[6])"#)); assert!(RESTART_SCRIPT.contains(r#"redis.call("SADD", KEYS[13], ARGV[10])"#)); + assert!(RESTART_SCRIPT.contains(r#"redis.call("ZREM", KEYS[16], ARGV[12])"#)); assert!(RESTART_SCRIPT.contains(r#"redis.call("ZADD", KEYS[14], ARGV[3], ARGV[11])"#)); + let lease_check = RESTART_SCRIPT.find("if not marker_score").unwrap(); + let first_write = RESTART_SCRIPT + .find(r#"redis.call("DEL", KEYS[2])"#) + .unwrap(); + assert!(lease_check < first_write); } #[test] diff --git a/rust/workflows/src/api/limits.rs b/rust/workflows/src/api/limits.rs index 5864c1e..caff1e0 100644 --- a/rust/workflows/src/api/limits.rs +++ b/rust/workflows/src/api/limits.rs @@ -1,6 +1,7 @@ pub(crate) const MAX_WORKFLOW_JSON_BODY_BYTES: usize = 2 * 1024 * 1024; pub(crate) const MAX_WORKFLOW_PARAMS_BYTES: usize = 1024 * 1024; pub(crate) const MAX_WORKFLOW_RESULT_BYTES: usize = 1024 * 1024; +pub(crate) const WORKFLOW_PAYLOAD_TOO_LARGE_CODE: &str = "workflow_payload_too_large"; pub(crate) const MAX_WORKFLOW_RUNTIME_RESPONSE_BYTES: usize = MAX_WORKFLOW_RESULT_BYTES + 16 * 1024; pub(crate) const MAX_WORKFLOW_EVENT_BYTES: usize = 256 * 1024; pub(crate) const MAX_WORKFLOW_EVENT_TYPE_BYTES: usize = 512; @@ -10,3 +11,41 @@ pub(crate) const MAX_WORKFLOW_STEP_CONFIG_BYTES: usize = 64 * 1024; pub(crate) const MAX_CREATE_BATCH_SIZE: usize = 100; pub(crate) const READY_SHARDS: usize = crate::WORKFLOW_READY_SHARDS; pub(crate) const LIFECYCLE_BLOCKER_LIMIT: usize = 20; + +#[cfg(test)] +mod tests { + use serde::Deserialize; + + use super::{ + MAX_CREATE_BATCH_SIZE, MAX_WORKFLOW_JSON_BODY_BYTES, MAX_WORKFLOW_RESULT_BYTES, + WORKFLOW_PAYLOAD_TOO_LARGE_CODE, + }; + + #[derive(Deserialize)] + #[serde(rename_all = "camelCase")] + struct WorkflowLimitsContract { + result_bytes_max: usize, + backend_request_bytes_max: usize, + create_batch_max: usize, + payload_too_large_code: String, + } + + #[test] + fn workflow_limits_match_runtime_fixture() { + let contract: WorkflowLimitsContract = serde_json::from_str(include_str!( + "../../../../tests/fixtures/workflow-limits.json" + )) + .expect("workflow limits fixture"); + + assert_eq!(contract.result_bytes_max, MAX_WORKFLOW_RESULT_BYTES); + assert_eq!( + contract.backend_request_bytes_max, + MAX_WORKFLOW_JSON_BODY_BYTES + ); + assert_eq!(contract.create_batch_max, MAX_CREATE_BATCH_SIZE); + assert_eq!( + contract.payload_too_large_code, + WORKFLOW_PAYLOAD_TOO_LARGE_CODE + ); + } +} diff --git a/rust/workflows/src/api/pending_restart.rs b/rust/workflows/src/api/pending_restart.rs new file mode 100644 index 0000000..1600e8e --- /dev/null +++ b/rust/workflows/src/api/pending_restart.rs @@ -0,0 +1,212 @@ +use wdl_rust_common::time::random_hex_64; + +use crate::{AppState, WorkflowError, WorkflowResult, pending_version_key}; + +use super::{LifecycleBlocker, WorkflowRequest, eval_script}; + +const PENDING_RESTART_TTL_MS: i64 = 30_000; +const PENDING_RESTART_KEY_TTL_MS: i64 = 60_000; + +const CREATE_PENDING_RESTART_SCRIPT: &str = r#" +local time = redis.call("TIME") +local now = tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000) +redis.call("ZREMRANGEBYSCORE", KEYS[1], "-inf", now) +redis.call("ZADD", KEYS[1], now + tonumber(ARGV[2]), ARGV[1]) +redis.call("PEXPIRE", KEYS[1], ARGV[3]) +return 1 +"#; + +const RENEW_PENDING_RESTART_SCRIPT: &str = r#" +local time = redis.call("TIME") +local now = tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000) +redis.call("ZREMRANGEBYSCORE", KEYS[1], "-inf", now) +local current = redis.call("ZSCORE", KEYS[1], ARGV[1]) +if not current then + return 0 +end +redis.call("ZADD", KEYS[1], now + tonumber(ARGV[2]), ARGV[1]) +redis.call("PEXPIRE", KEYS[1], ARGV[3]) +return 1 +"#; + +const READ_PENDING_RESTART_SCRIPT: &str = r#" +local time = redis.call("TIME") +local now = tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000) +if ARGV[1] == "1" then + redis.call("ZREMRANGEBYSCORE", KEYS[1], "-inf", now) +end +local active_min = "(" .. tostring(now) +local count = redis.call("ZCOUNT", KEYS[1], active_min, "+inf") +local members = redis.call("ZRANGEBYSCORE", KEYS[1], active_min, "+inf", "LIMIT", 0, ARGV[2]) +return { count, members } +"#; + +pub(super) struct PendingRestartMarker { + pub(super) key: String, + pub(super) member: String, +} + +pub(super) fn pending_restart_marker( + state: &AppState, + req: &WorkflowRequest, + instance_id: &str, +) -> PendingRestartMarker { + PendingRestartMarker { + key: pending_version_key(&req.ns, &req.worker, &req.frozen_version), + member: format!( + "{}\t{}\t{}:{}", + req.workflow_key, + instance_id, + state.instance_id, + random_hex_64() + ), + } +} + +pub(super) async fn create_pending_restart( + state: &AppState, + marker: &PendingRestartMarker, +) -> WorkflowResult<()> { + let marker_ttl = PENDING_RESTART_TTL_MS.to_string(); + let key_ttl = PENDING_RESTART_KEY_TTL_MS.to_string(); + let _: i64 = eval_script( + state, + CREATE_PENDING_RESTART_SCRIPT, + &[&marker.key], + &[&marker.member, &marker_ttl, &key_ttl], + ) + .await?; + Ok(()) +} + +pub(super) async fn renew_pending_restart( + state: &AppState, + marker: &PendingRestartMarker, +) -> WorkflowResult { + let marker_ttl = PENDING_RESTART_TTL_MS.to_string(); + let key_ttl = PENDING_RESTART_KEY_TTL_MS.to_string(); + let renewed: i64 = eval_script( + state, + RENEW_PENDING_RESTART_SCRIPT, + &[&marker.key], + &[&marker.member, &marker_ttl, &key_ttl], + ) + .await?; + Ok(renewed == 1) +} + +pub(super) async fn remove_pending_restart( + state: &AppState, + marker: &PendingRestartMarker, +) -> WorkflowResult<()> { + let key = marker.key.clone(); + let member = marker.member.clone(); + state + .redis + .with_conn(async move |mut conn| { + redis::cmd("ZREM") + .arg(key) + .arg(member) + .query_async::<()>(&mut conn) + .await + }) + .await?; + Ok(()) +} + +pub(super) async fn active_pending_restart_blockers( + state: &AppState, + ns: &str, + worker: &str, + version: &str, + allow_cleanup: bool, + limit: usize, +) -> WorkflowResult<(usize, Vec)> { + let key = pending_version_key(ns, worker, version); + let cleanup = if allow_cleanup { "1" } else { "0" }; + let limit = limit.to_string(); + let (count, members): (usize, Vec) = eval_script( + state, + READ_PENDING_RESTART_SCRIPT, + &[&key], + &[cleanup, &limit], + ) + .await?; + let blockers = members + .into_iter() + .map(|member| parse_pending_restart_member(&member)) + .collect::>>()?; + Ok((count, blockers)) +} + +fn parse_pending_restart_member(member: &str) -> WorkflowResult { + let mut parts = member.split('\t'); + let workflow_key = parts.next().unwrap_or_default(); + let instance_id = parts.next().unwrap_or_default(); + let token = parts.next().unwrap_or_default(); + if workflow_key.is_empty() + || instance_id.is_empty() + || token.is_empty() + || parts.next().is_some() + { + return Err(WorkflowError::invalid_state( + "Pending workflow restart blocker is corrupt", + )); + } + Ok(LifecycleBlocker { + workflow_key: workflow_key.to_string(), + instance_id: instance_id.to_string(), + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pending_restart_member_preserves_blocker_identity() { + let blocker = parse_pending_restart_member("wf_key\tinstance-1\ttask:token").unwrap(); + assert_eq!(blocker.workflow_key, "wf_key"); + assert_eq!(blocker.instance_id, "instance-1"); + assert_eq!(PENDING_RESTART_TTL_MS, 30_000); + assert_eq!(PENDING_RESTART_KEY_TTL_MS, 60_000); + } + + #[test] + fn pending_restart_member_rejects_malformed_state() { + for member in [ + "", + "wf_key", + "wf_key\tinstance-1", + "wf\tinstance\ttoken\textra", + ] { + assert!(parse_pending_restart_member(member).is_err()); + } + } + + #[test] + fn pending_restart_scripts_use_redis_time_and_do_not_resurrect_expired_members() { + assert!(CREATE_PENDING_RESTART_SCRIPT.contains(r#"redis.call("TIME")"#)); + assert!( + CREATE_PENDING_RESTART_SCRIPT.contains(r#"redis.call("PEXPIRE", KEYS[1], ARGV[3])"#) + ); + assert!( + RENEW_PENDING_RESTART_SCRIPT + .contains(r#"local current = redis.call("ZSCORE", KEYS[1], ARGV[1])"#) + ); + assert!(RENEW_PENDING_RESTART_SCRIPT.contains("if not current then")); + assert!(READ_PENDING_RESTART_SCRIPT.contains(r#"redis.call("TIME")"#)); + assert!(READ_PENDING_RESTART_SCRIPT.contains("return { count, members }")); + assert!(CREATE_PENDING_RESTART_SCRIPT.contains("ZREMRANGEBYSCORE")); + assert!(RENEW_PENDING_RESTART_SCRIPT.contains("ZREMRANGEBYSCORE")); + let cleanup = RENEW_PENDING_RESTART_SCRIPT + .find("ZREMRANGEBYSCORE") + .unwrap(); + let expiry_check = RENEW_PENDING_RESTART_SCRIPT.find("if not current").unwrap(); + let write = RENEW_PENDING_RESTART_SCRIPT + .find(r#"redis.call("ZADD""#) + .unwrap(); + assert!(cleanup < expiry_check); + assert!(expiry_check < write); + } +} diff --git a/rust/workflows/src/api/progress.rs b/rust/workflows/src/api/progress.rs index ed4f67e..ed6a958 100644 --- a/rust/workflows/src/api/progress.rs +++ b/rust/workflows/src/api/progress.rs @@ -1,4 +1,4 @@ -use super::{execution::WorkflowStepRequest, parse_positive_identity_i64}; +use super::{execution::WorkflowStepRequest, parse_positive_identity_i64, runtime_endpoint}; use serde_json::{Value as JsonValue, json}; use wdl_rust_common::internal_auth::INTERNAL_AUTH_HEADER; use wdl_rust_common::time::now_ms; @@ -9,18 +9,6 @@ use crate::{ const PROGRESS_CALLBACK_CACHE_LIMIT: usize = 4096; -fn runtime_endpoint(app: &AppState, ns: &str) -> String { - let (host, port) = if ns == "__system__" { - ( - &app.config.system_runtime_host, - app.config.system_runtime_port, - ) - } else { - (&app.config.runtime_host, app.config.runtime_port) - }; - format!("http://{host}:{port}/internal/workflows/notify") -} - fn progress_payload(event: &str, status: &str, step: Option) -> JsonValue { json!({ "event": event, @@ -83,7 +71,7 @@ async fn notify(app: &AppState, identity: InstanceIdentity, callback: String, pr ); return; }; - let url = runtime_endpoint(app, &identity.ns); + let url = runtime_endpoint(app, &identity.ns, "/internal/workflows/notify"); let response = app .http .post(url) diff --git a/rust/workflows/src/api/tick/dispatch.rs b/rust/workflows/src/api/tick/dispatch.rs index 2627b14..a0be147 100644 --- a/rust/workflows/src/api/tick/dispatch.rs +++ b/rust/workflows/src/api/tick/dispatch.rs @@ -10,10 +10,11 @@ use crate::{ }; use super::super::{ - InstanceRouteKeys, MAX_WORKFLOW_RUNTIME_RESPONSE_BYTES, RunClaim, clear_suspended_run_claim, - eval_script, instance_payload_limit_arg, log_instance_event, observe_instance_duration, - parse_positive_identity_i64, read_state_by_id, result_json, spawn_progress_from_identity, - terminal_retention_ms, + InstanceRouteKeys, MAX_WORKFLOW_RUNTIME_RESPONSE_BYTES, RunClaim, + WORKFLOW_PAYLOAD_TOO_LARGE_CODE, clear_suspended_run_claim, eval_script, + instance_payload_limit_arg, log_instance_event, observe_instance_duration, + parse_positive_identity_i64, read_state_by_id, result_json, runtime_endpoint, + spawn_progress_from_identity, terminal_retention_ms, }; pub(crate) const COMMIT_RUNTIME_TERMINAL_SCRIPT: &str = r#" @@ -226,17 +227,7 @@ pub(super) async fn dispatch_runtime( params: JsonValue, request_id: Option<&str>, ) -> WorkflowResult { - let host = if identity.ns == "__system__" { - &app.config.system_runtime_host - } else { - &app.config.runtime_host - }; - let port = if identity.ns == "__system__" { - app.config.system_runtime_port - } else { - app.config.runtime_port - }; - let url = format!("http://{host}:{port}/internal/workflows/run"); + let url = runtime_endpoint(app, &identity.ns, "/internal/workflows/run"); let generation = parse_positive_identity_i64(&identity.generation, "generation")?; let created_at_ms = parse_positive_identity_i64(&identity.created_at_ms, "createdAtMs")?; let mut request = app @@ -329,7 +320,7 @@ async fn commit_runtime_failed_payload( fn runtime_payload_too_large_error(err: &WorkflowError) -> JsonValue { json!({ "name": "WorkflowError", - "code": "workflow_payload_too_large", + "code": WORKFLOW_PAYLOAD_TOO_LARGE_CODE, "message": err.message, }) } diff --git a/rust/workflows/src/keys.rs b/rust/workflows/src/keys.rs index faf7dc1..9738af4 100644 --- a/rust/workflows/src/keys.rs +++ b/rust/workflows/src/keys.rs @@ -162,6 +162,10 @@ pub(crate) fn by_version_key(ns: &str, worker: &str, version: &str) -> String { format!("wf:by-version:{ns}:{worker}:{version}") } +pub(crate) fn pending_version_key(ns: &str, worker: &str, version: &str) -> String { + format!("wf:pending-version:{ns}:{worker}:{version}") +} + pub(crate) fn retention_key() -> &'static str { "wf:retention" } diff --git a/rust/workflows/src/server.rs b/rust/workflows/src/server.rs index 9569566..1f64e18 100644 --- a/rust/workflows/src/server.rs +++ b/rust/workflows/src/server.rs @@ -25,9 +25,12 @@ use serde::Serialize; use serde_json::{Value as JsonValue, json}; use tokio::sync::Semaphore; use wdl_rust_common::health::healthcheck_http_200; -use wdl_rust_common::internal_auth::internal_auth_headers_match; +use wdl_rust_common::internal_auth::{ + INTERNAL_AUTH_FAILURE_CODE, INTERNAL_AUTH_FAILURE_MESSAGE, internal_auth_failure_response, + internal_auth_headers_match, +}; use wdl_rust_common::metrics::prometheus_response; -use wdl_rust_common::request_id::sanitize_request_id; +use wdl_rust_common::request_id::request_id_from_headers; use wdl_rust_common::shutdown::shutdown_signal; use wdl_rust_common::time::duration_ms_for_log; @@ -371,13 +374,6 @@ fn response_error(response: &Response) -> Option<(&'static str, &str)> { .map(|err| (err.code, err.message.as_str())) } -fn request_id_from_headers(headers: &HeaderMap) -> Option { - headers - .get("x-request-id") - .and_then(|value| value.to_str().ok()) - .and_then(sanitize_request_id) -} - async fn track_request( State(state): State, request: Request, @@ -390,14 +386,7 @@ async fn track_request( if !matches!(route, "healthz" | "metrics") && !internal_auth_headers_match(request.headers(), &state.config.internal_auth_tokens) { - let response = ( - StatusCode::UNAUTHORIZED, - Json(json!({ - "error": "internal_auth_failed", - "message": "Internal authentication failed", - })), - ) - .into_response(); + let response = internal_auth_failure_response(); record_request_complete( &state, method.as_str(), @@ -405,7 +394,7 @@ async fn track_request( response.status(), request_id.as_deref(), started_at, - Some(("internal_auth_failed", "Internal authentication failed")), + Some((INTERNAL_AUTH_FAILURE_CODE, INTERNAL_AUTH_FAILURE_MESSAGE)), ); return response; } diff --git a/rust/workflows/src/tests.rs b/rust/workflows/src/tests.rs index 2cfe33c..79e1d24 100644 --- a/rust/workflows/src/tests.rs +++ b/rust/workflows/src/tests.rs @@ -4,25 +4,16 @@ use super::{ }; use crate::api::{canonical_json, retry_due_at_ms, retry_policy}; use crate::{ - InstanceKeys, config_from_env, due_key, ready_active_key, ready_key, schema_version_key, - validate_instance_id_value, + InstanceKeys, config_from_env, due_key, pending_version_key, ready_active_key, ready_key, + schema_version_key, validate_instance_id_value, }; -use std::sync::{Mutex, OnceLock}; - -static ENV_LOCK: OnceLock> = OnceLock::new(); +use wdl_rust_common::test_env::with_temp_envs; // Source-contract tests below intentionally couple to these module paths. If // the files move, update the constants and the associated assertions together. const RUNTIME_DISPATCH_SOURCE: &str = include_str!("api/tick/dispatch.rs"); const EXPECTED_EVENT_INDEX_STALE_SCAN_LIMIT: usize = 256; fn temp_env(items: &[(&str, Option<&str>)], f: impl FnOnce() -> R) -> R { - // Environment variables are process-global. This helper only protects the - // workflows tests that opt into it; tests that read or mutate env - // directly must stay out of the same parallel window. - let _guard = ENV_LOCK - .get_or_init(|| Mutex::new(())) - .lock() - .expect("env mutex poisoned"); let mut all_items = items.to_vec(); if !all_items .iter() @@ -30,31 +21,7 @@ fn temp_env(items: &[(&str, Option<&str>)], f: impl FnOnce() -> R) -> R { { all_items.push(("WDL_INTERNAL_AUTH_TOKEN", Some("test-internal-auth-token"))); } - let previous: Vec<(&str, Option)> = all_items - .iter() - .map(|(key, _)| (*key, std::env::var(key).ok())) - .collect(); - for (key, value) in &all_items { - match value { - // SAFETY: Rust marks environment mutation unsafe because other - // threads can read concurrently. ENV_LOCK serializes this test - // helper's callers, and the workflows env tests keep all - // config reads inside the locked closure. - Some(value) => unsafe { std::env::set_var(key, value) }, - // SAFETY: Same rationale as set_var above. - None => unsafe { std::env::remove_var(key) }, - } - } - let result = f(); - for (key, value) in previous { - match value { - // SAFETY: Same rationale as the initial set_var above. - Some(value) => unsafe { std::env::set_var(key, value) }, - // SAFETY: Same rationale as set_var above. - None => unsafe { std::env::remove_var(key) }, - } - } - result + with_temp_envs(&all_items, f) } #[test] @@ -320,6 +287,14 @@ fn workflow_ready_key_uses_fixed_shards() { assert_eq!(ready_key(31), "wf:ready:31"); } +#[test] +fn pending_restart_key_is_scoped_to_the_target_version() { + assert_eq!( + pending_version_key("demo", "orders", "v3"), + "wf:pending-version:demo:orders:v3" + ); +} + #[test] fn workflow_schema_version_key_is_global_db2_marker() { assert_eq!(schema_version_key(), "wf:schema_version"); diff --git a/scripts/_integration-pool.js b/scripts/_integration-pool.js index f0f659c..0ceb801 100644 --- a/scripts/_integration-pool.js +++ b/scripts/_integration-pool.js @@ -1,17 +1,21 @@ import { spawn } from "node:child_process"; -import { existsSync, readFileSync, renameSync, writeFileSync } from "node:fs"; +import { renameSync, writeFileSync } from "node:fs"; import { COMPILE_WORKERD_LOCAL_ARGS, DOCKER_COMPOSE_BUILD_ARGS, + LOCAL_ADMIN_TOKEN, + LOCAL_CONNECT_HOST, ROOT, + localAssetsCdnBase, + localControlUrl, shouldPrepareIntegrationArtifacts, } from "./integration-environment.js"; +import { readIntegrationDurationReport } from "./integration-test-plan.js"; +import { pipeWithPrefix, writeWithPrefix } from "./_stream-prefix.js"; /** * @typedef {{ file: string, slot: number, durationMs: number, status: "passed" | "failed" }} FileDuration * @typedef {{ shardCount: number, runDurationMs: number, fileDurations: FileDuration[] }} IntegrationSummary - * @typedef {{ durationMs: number, status: string, updatedAt: string }} DurationRecord - * @typedef {{ updatedAt?: string, runDurationMs?: number, files?: Record }} DurationReport * @typedef {{ * files: string[], * shardCount: number, @@ -215,10 +219,10 @@ export function makeSlotEnv(idx, projectPrefix, basePort, s3PortBase, baseEnv = WDL_S3MOCK_HOST_PORT: String(s3mockPort), WDL_WORKERD_CONFIG_VARIANT: "local", WDL_INTEGRATION_NO_BUILD: "1", - ADMIN_TOKEN: "local-dev-token", - CONTROL_URL: `http://admin.test:${gatewayPort}`, - ASSETS_CDN_BASE: `http://localhost:${s3mockPort}/wdl-assets`, - CONTROL_CONNECT_HOST: "localhost", + ADMIN_TOKEN: LOCAL_ADMIN_TOKEN, + CONTROL_URL: localControlUrl(gatewayPort), + ASSETS_CDN_BASE: localAssetsCdnBase(s3mockPort), + CONTROL_CONNECT_HOST: LOCAL_CONNECT_HOST, }; return env; } @@ -251,8 +255,9 @@ function teardownSlot(env, prefix) { }); } -/** @param {DurationReport | null | undefined} existing @param {{ runDurationMs: number, fileDurations: FileDuration[] }} observed @param {Date} [now] */ +/** @param {ReturnType | undefined} existing @param {{ runDurationMs: number, fileDurations: FileDuration[] }} observed @param {Date} [now] */ export function buildDurationReport(existing, { runDurationMs, fileDurations }, now = new Date()) { + /** @type {Record} */ const files = { ...(existing?.files && typeof existing.files === "object" ? existing.files : {}) }; const updatedAt = now.toISOString(); for (const row of fileDurations) { @@ -315,44 +320,9 @@ export function persistDurationFile(file, observed) { /** @param {string} file @param {{ runDurationMs: number, fileDurations: FileDuration[] }} observed */ function updateDurationFile(file, observed) { - const existing = readDurationFile(file); + const existing = readIntegrationDurationReport(file); const next = buildDurationReport(existing, observed); const tmp = `${file}.tmp-${process.pid}`; writeFileSync(tmp, `${JSON.stringify(next, null, 2)}\n`); renameSync(tmp, file); } - -/** @param {string} file @returns {DurationReport} */ -function readDurationFile(file) { - if (!existsSync(file)) return {}; - try { - return JSON.parse(readFileSync(file, "utf8")); - } catch (err) { - process.stderr.write( - `warning: ignoring unreadable integration duration file ${file}: ${errorMessage(err)}\n` - ); - return {}; - } -} - -/** @param {NodeJS.ReadableStream} src @param {NodeJS.WritableStream} dst @param {string} prefix */ -function pipeWithPrefix(src, dst, prefix) { - let buf = ""; - src.setEncoding("utf8"); - src.on("data", (chunk) => { - buf += String(chunk); - let nl; - while ((nl = buf.indexOf("\n")) >= 0) { - writeWithPrefix(dst, prefix, `${buf.slice(0, nl)}\n`); - buf = buf.slice(nl + 1); - } - }); - src.on("end", () => { - if (buf.length) writeWithPrefix(dst, prefix, `${buf}\n`); - }); -} - -/** @param {NodeJS.WritableStream} dst @param {string} prefix @param {string} text */ -function writeWithPrefix(dst, prefix, text) { - dst.write(`${prefix}${text}`); -} diff --git a/scripts/_stream-prefix.js b/scripts/_stream-prefix.js new file mode 100644 index 0000000..89d7669 --- /dev/null +++ b/scripts/_stream-prefix.js @@ -0,0 +1,21 @@ +/** @param {NodeJS.WritableStream} dst @param {string} prefix @param {string} text */ +export function writeWithPrefix(dst, prefix, text) { + dst.write(`${prefix}${text}`); +} + +/** @param {NodeJS.ReadableStream} src @param {NodeJS.WritableStream} dst @param {string} prefix */ +export function pipeWithPrefix(src, dst, prefix) { + let buffer = ""; + src.setEncoding("utf8"); + src.on("data", (chunk) => { + buffer += String(chunk); + let newline; + while ((newline = buffer.indexOf("\n")) >= 0) { + writeWithPrefix(dst, prefix, `${buffer.slice(0, newline)}\n`); + buffer = buffer.slice(newline + 1); + } + }); + src.on("end", () => { + if (buffer.length) writeWithPrefix(dst, prefix, `${buffer}\n`); + }); +} diff --git a/scripts/integration-environment.js b/scripts/integration-environment.js index ed377f3..5b96585 100644 --- a/scripts/integration-environment.js +++ b/scripts/integration-environment.js @@ -5,6 +5,19 @@ export const ROOT = path.resolve(import.meta.dirname, ".."); export const COMPILE_WORKERD_LOCAL_ARGS = ["scripts/compile-workerd-configs.js", "--local"]; export const DOCKER_COMPOSE_BUILD_ARGS = ["compose", "build", "gateway", "workflows"]; export const DEFAULT_WDL_CLI_COMMAND = "wdl"; +export const LOCAL_ADMIN_TOKEN = "local-dev-token"; +export const ADMIN_HOST_HEADER = "admin.test"; +export const LOCAL_CONNECT_HOST = "localhost"; + +/** @param {number} gatewayPort */ +export function localControlUrl(gatewayPort) { + return `http://${ADMIN_HOST_HEADER}:${gatewayPort}`; +} + +/** @param {number} s3mockPort */ +export function localAssetsCdnBase(s3mockPort) { + return `http://${LOCAL_CONNECT_HOST}:${s3mockPort}/wdl-assets`; +} /** @param {string} file */ function isExecutableFile(file) { diff --git a/scripts/integration-test-plan.js b/scripts/integration-test-plan.js index 9a2697a..fabd257 100644 --- a/scripts/integration-test-plan.js +++ b/scripts/integration-test-plan.js @@ -74,6 +74,7 @@ const CLI_INTEGRATION_MARKER_RE = new RegExp(`^//\\s*${RegExp.escape(CLI_INTEGRA /** * @typedef {{ durationMs?: unknown, status?: unknown, updatedAt?: unknown }} DurationRecord * @typedef {Record} DurationRecords + * @typedef {{ updatedAt?: unknown, runDurationMs?: unknown, files?: unknown }} DurationReport * @typedef {{ priority?: string[], durationRecords?: DurationRecords | null }} PrioritizeOptions */ @@ -82,21 +83,22 @@ function errorMessage(err) { return err instanceof Error ? err.message : String(err); } -/** @param {unknown} value @returns {DurationRecords | null} */ -function durationRecordsFromJson(value) { - if (!value || typeof value !== "object") return null; - const files = /** @type {{ files?: unknown }} */ (value).files; +/** @param {DurationReport | null} report @returns {DurationRecords | null} */ +function durationRecordsFromReport(report) { + const files = report?.files; return files && typeof files === "object" ? /** @type {DurationRecords} */ (files) : null; } -/** @param {string} [file] @returns {DurationRecords | null} */ -export function readIntegrationDurationRecords(file = DEFAULT_INTEGRATION_DURATIONS_FILE) { +/** @param {string} [file] @returns {DurationReport | null} */ +export function readIntegrationDurationReport(file = DEFAULT_INTEGRATION_DURATIONS_FILE) { if (!file || !existsSync(file)) return null; try { const parsed = JSON.parse(readFileSync(file, "utf8")); - return durationRecordsFromJson(parsed); + return parsed && typeof parsed === "object" + ? /** @type {DurationReport} */ (parsed) + : null; } catch (err) { process.stderr.write( `warning: ignoring unreadable integration duration file ${file}: ${errorMessage(err)}\n` @@ -105,6 +107,11 @@ export function readIntegrationDurationRecords(file = DEFAULT_INTEGRATION_DURATI } } +/** @param {string} [file] @returns {DurationRecords | null} */ +export function readIntegrationDurationRecords(file = DEFAULT_INTEGRATION_DURATIONS_FILE) { + return durationRecordsFromReport(readIntegrationDurationReport(file)); +} + /** @param {string[]} names @param {DurationRecords | null | undefined} durationRecords */ export function durationPriorityNames(names, durationRecords) { if (!durationRecords) return []; diff --git a/scripts/run-parallel.js b/scripts/run-parallel.js index 1192f97..2eeafe4 100644 --- a/scripts/run-parallel.js +++ b/scripts/run-parallel.js @@ -1,21 +1,5 @@ import { spawn } from "node:child_process"; - -/** @param {NodeJS.ReadableStream} src @param {NodeJS.WritableStream} dst @param {string} prefix */ -function pipeWithPrefix(src, dst, prefix) { - let buf = ""; - src.setEncoding("utf8"); - src.on("data", (chunk) => { - buf += String(chunk); - let nl; - while ((nl = buf.indexOf("\n")) >= 0) { - dst.write(`${prefix}${buf.slice(0, nl)}\n`); - buf = buf.slice(nl + 1); - } - }); - src.on("end", () => { - if (buf.length) dst.write(`${prefix}${buf}\n`); - }); -} +import { pipeWithPrefix } from "./_stream-prefix.js"; const tasks = process.argv.slice(2); if (tasks.length === 0) { diff --git a/shared/base64.js b/shared/base64.js index 9d49ff4..004b01b 100644 --- a/shared/base64.js +++ b/shared/base64.js @@ -3,8 +3,29 @@ // needed. const CHUNK_SIZE = 0x8000; +const BASE64_ALPHABET_RE = /^[A-Za-z0-9+/]*$/; +const BASE64_WHITESPACE_RE = /[\t\n\f\r ]/; +const BASE64_WHITESPACE_GLOBAL_RE = /[\t\n\f\r ]/g; const utf8Encoder = new TextEncoder(); +// Buffer.from(..., "base64") skips invalid bytes and accepts base64url. +// Validate with the same forgiving-base64 grammar as atob first so the +// nodejs_compat and web-platform branches fail closed identically. +/** @param {string} value */ +function validatedBufferBase64(value) { + let payload = BASE64_WHITESPACE_RE.test(value) + ? value.replace(BASE64_WHITESPACE_GLOBAL_RE, "") + : value; + if (payload.length % 4 === 0) { + if (payload.endsWith("==")) payload = payload.slice(0, -2); + else if (payload.endsWith("=")) payload = payload.slice(0, -1); + } + if (payload.length % 4 === 1 || !BASE64_ALPHABET_RE.test(payload)) { + throw new TypeError("Invalid base64 input"); + } + return payload; +} + /** @param {Uint8Array} bytes @returns {string} */ export function bytesToBase64(bytes) { if (typeof Buffer !== "undefined") return Buffer.from(bytes).toString("base64"); @@ -17,7 +38,9 @@ export function bytesToBase64(bytes) { /** @param {string} value @returns {Uint8Array} */ export function base64ToBytes(value) { - if (typeof Buffer !== "undefined") return new Uint8Array(Buffer.from(value, "base64")); + if (typeof Buffer !== "undefined") { + return new Uint8Array(Buffer.from(validatedBufferBase64(value), "base64")); + } const binary = atob(value); const out = new Uint8Array(binary.length); for (let i = 0; i < binary.length; i += 1) out[i] = binary.charCodeAt(i); diff --git a/shared/bounded-body.js b/shared/bounded-body.js index 4116f1d..112ca12 100644 --- a/shared/bounded-body.js +++ b/shared/bounded-body.js @@ -10,21 +10,17 @@ export class BodyTooLargeError extends Error { } /** - * @param {Request} request + * @param {ReadableStream} stream * @param {number} maxBytes + * @param {() => Error} [overflowError] * @returns {Promise} */ -export async function readBoundedBytes(request, maxBytes) { - const contentLength = request.headers.get("content-length"); - if (contentLength != null && contentLength !== "") { - const declared = Number(contentLength); - if (Number.isFinite(declared) && declared > maxBytes) { - throw new BodyTooLargeError(maxBytes); - } - } - if (!request.body) return new Uint8Array(); - - const reader = request.body.getReader(); +export async function readBoundedStreamBytes( + stream, + maxBytes, + overflowError = () => new BodyTooLargeError(maxBytes) +) { + const reader = stream.getReader(); /** @type {Uint8Array[]} */ const chunks = []; let total = 0; @@ -32,17 +28,28 @@ export async function readBoundedBytes(request, maxBytes) { while (true) { const { done, value } = await reader.read(); if (done) break; - total += value.byteLength; + const chunk = value instanceof Uint8Array ? value : new Uint8Array(value); + total += chunk.byteLength; if (total > maxBytes) { - await reader.cancel().catch(() => {}); - throw new BodyTooLargeError(maxBytes); + const error = overflowError(); + try { + void reader.cancel(error).catch(() => {}); + } catch { + // Cancellation is best-effort; the size error must not wait on it. + } + throw error; } - chunks.push(value); + chunks.push(chunk); } } finally { - reader.releaseLock(); + try { reader.releaseLock(); } catch {} } + if (chunks.length === 1) { + const [chunk] = chunks; + if (chunk.byteOffset === 0 && chunk.byteLength === chunk.buffer.byteLength) return chunk; + return new Uint8Array(chunk); + } const bytes = new Uint8Array(total); let offset = 0; for (const chunk of chunks) { @@ -52,6 +59,23 @@ export async function readBoundedBytes(request, maxBytes) { return bytes; } +/** + * @param {Request} request + * @param {number} maxBytes + * @returns {Promise} + */ +export async function readBoundedBytes(request, maxBytes) { + const contentLength = request.headers.get("content-length"); + if (contentLength != null && contentLength !== "") { + const declared = Number(contentLength); + if (Number.isFinite(declared) && declared > maxBytes) { + throw new BodyTooLargeError(maxBytes); + } + } + if (!request.body) return new Uint8Array(); + return readBoundedStreamBytes(request.body, maxBytes); +} + /** * @param {Request} request * @param {number} maxBytes diff --git a/shared/d1-transport.js b/shared/d1-transport.js index c3f038e..66fe95b 100644 --- a/shared/d1-transport.js +++ b/shared/d1-transport.js @@ -110,7 +110,6 @@ function walkD1Transport(value, objectLeaf) { /** * @param {unknown} value * @returns {unknown} - * @lintignore runtime/_wdl-d1-transport.js re-exports this for loaded workers. */ export function decodeD1Transport(value) { return walkD1Transport(value, (item) => { diff --git a/shared/errors.js b/shared/errors.js index 783c9b9..015e6a1 100644 --- a/shared/errors.js +++ b/shared/errors.js @@ -1,4 +1,8 @@ /** @param {unknown} err */ export function errorMessage(err) { - return err instanceof Error ? err.message : String(err); + try { + return String(err instanceof Error ? err.message : err); + } catch { + return "Unknown error"; + } } diff --git a/shared/internal-auth.js b/shared/internal-auth.js index 692f4f7..defc864 100644 --- a/shared/internal-auth.js +++ b/shared/internal-auth.js @@ -1,18 +1,21 @@ export const INTERNAL_AUTH_HEADER = "x-wdl-internal-auth"; export const INTERNAL_AUTH_ENV = "WDL_INTERNAL_AUTH_TOKEN"; export const INTERNAL_AUTH_PREVIOUS_ENV = "WDL_INTERNAL_AUTH_PREVIOUS_TOKEN"; +export const INTERNAL_AUTH_FAILURE_CODE = "internal_auth_failed"; +export const INTERNAL_AUTH_FAILURE_MESSAGE = "Internal authentication failed"; /** * @param {string} name * @param {string} token */ -function assertAsciiToken(name, token) { +function assertHeaderToken(name, token) { if (token === "") { - throw new Error(`${name} must be configured as a non-empty ASCII string`); + throw new Error(`${name} must be configured as visible ASCII without whitespace or commas`); } for (let i = 0; i < token.length; i++) { - if (token.charCodeAt(i) > 127) { - throw new Error(`${name} must be configured as a non-empty ASCII string`); + const code = token.charCodeAt(i); + if (code < 0x21 || code > 0x7e || code === 0x2c) { + throw new Error(`${name} must be configured as visible ASCII without whitespace or commas`); } } return token; @@ -26,7 +29,7 @@ export function internalAuthToken(env) { if (typeof token !== "string" || token === "") { throw new Error(`${INTERNAL_AUTH_ENV} must be configured`); } - return assertAsciiToken(INTERNAL_AUTH_ENV, token); + return assertHeaderToken(INTERNAL_AUTH_ENV, token); } /** @@ -34,7 +37,7 @@ export function internalAuthToken(env) { */ export function internalAuthPreviousToken(env) { const token = env?.[INTERNAL_AUTH_PREVIOUS_ENV]; - return typeof token === "string" && token !== "" ? assertAsciiToken(INTERNAL_AUTH_PREVIOUS_ENV, token) : null; + return typeof token === "string" && token !== "" ? assertHeaderToken(INTERNAL_AUTH_PREVIOUS_ENV, token) : null; } /** @@ -117,7 +120,7 @@ export function stripInternalAuthHeader(headers) { export function internalAuthFailureResponse() { return Response.json({ - error: "internal_auth_failed", - message: "Internal authentication failed", + error: INTERNAL_AUTH_FAILURE_CODE, + message: INTERNAL_AUTH_FAILURE_MESSAGE, }, { status: 401 }); } diff --git a/shared/ns-pattern.js b/shared/ns-pattern.js index f47cb1e..ac954e8 100644 --- a/shared/ns-pattern.js +++ b/shared/ns-pattern.js @@ -5,6 +5,42 @@ // character set can never drift between the tier that accepts deploys and // the tier that routes traffic. export const NS_PATTERN = "[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?"; +export const DEFAULT_PLATFORM_DOMAIN = "workers.local"; +export const MAX_PLATFORM_DOMAIN_BYTES = 126; +const DNS_LABEL_RE = /^[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?$/; +const DNS_FINAL_LABEL_RE = /^[A-Za-z]+$/; + +/** @param {string} value */ +function isAscii(value) { + for (let index = 0; index < value.length; index += 1) { + if (value.charCodeAt(index) > 0x7f) return false; + } + return true; +} + +/** @param {unknown} value */ +export function configuredHostname(value) { + if (typeof value !== "string" || !value.trim()) return null; + const trimmed = value.trim(); + const rawHost = trimmed.endsWith(".") ? trimmed.slice(0, -1) : trimmed; + if (!isAscii(rawHost)) return null; + if (rawHost.length === 0 || rawHost.length > MAX_PLATFORM_DOMAIN_BYTES) return null; + const labels = rawHost.split("."); + if ( + labels.some((label) => !DNS_LABEL_RE.test(label)) || + !DNS_FINAL_LABEL_RE.test(labels.at(-1) || "") + ) return null; + return rawHost.toLowerCase(); +} + +/** @param {Record} env */ +export function platformDomainFromEnv(env) { + const raw = env.PLATFORM_DOMAIN; + if (raw == null || raw === "" || typeof raw !== "string") return DEFAULT_PLATFORM_DOMAIN; + const domain = configuredHostname(raw); + if (!domain) throw new TypeError("PLATFORM_DOMAIN must be an ALB-compatible ASCII DNS hostname"); + return domain; +} export const RESERVED_NS = new Set(["__system__", "__platform__", "__community__"]); @@ -96,6 +132,7 @@ export const WORKFLOW_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/; // Workflow instance ids are embedded into DB2 keys and tab-delimited scheduler // tokens. Keep them URL/key friendly and delimiter-free. export const WORKFLOW_INSTANCE_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/; +export const WORKFLOW_KEY_RE = /^wf_[0-9a-f]{32}$/; // `:` in an id would corrupt `queue:::s` parsing; camelCase // would split one logical queue across two log-field entries. @@ -111,6 +148,11 @@ export const BINDING_NAME_RE = /^[A-Za-z_$][A-Za-z0-9_$]{0,63}$/; // while binding names add a platform cap for env/log hygiene. export const JS_IDENTIFIER_RE = /^[A-Za-z_$][A-Za-z0-9_$]*$/; +// DO host ids are `<35-byte storage id>::shard15` and may not exceed +// 512 UTF-8 bytes. Class declarations use ASCII grammar, so 468 characters is +// the deploy-time bound that keeps every shard addressable. +export const MAX_DO_CLASS_NAME_BYTES = 468; + /** * @param {unknown} value * @returns {boolean} @@ -193,6 +235,10 @@ export const WDL_RESERVED_ENTRYPOINT_RE = /^__Wdl[A-Za-z0-9_]*__$/; // key="bar") and (id="foo", key="v:bar"). export const KV_ID_RE = /^[a-z0-9][a-z0-9-]{0,62}$/; +// D1 API ids are stable control-plane references and may retain the +// mixed-case/underscore forms accepted by the existing D1 model. +export const D1_DATABASE_ID_RE = /^[A-Za-z0-9][A-Za-z0-9_-]{0,127}$/; + // Worker-scoped R2 virtual bucket name. Same Redis/S3-key hygiene class as // KV / Queue ids: lowercase, no slash, no colon, bounded length. export const R2_BUCKET_NAME_RE = /^[a-z0-9][a-z0-9-]{0,62}$/; diff --git a/shared/observability.js b/shared/observability.js index 4e58b6f..1993d31 100644 --- a/shared/observability.js +++ b/shared/observability.js @@ -4,6 +4,7 @@ // Label discipline + cardinality rules: see CLAUDE.md. import { bytesToHex } from "./hex.js"; +import { errorMessage } from "./errors.js"; const CARDINALITY_WARN_LIMIT = 100; /** @@ -119,10 +120,21 @@ export function generateRequestId() { return bytesToHex(bytes); } +/** @param {string} value */ +function trimAsciiWhitespace(value) { + let start = 0; + let end = value.length; + /** @param {number} code */ + const isWhitespace = (code) => code === 0x20 || code === 0x09 || code === 0x0a || code === 0x0c || code === 0x0d; + while (start < end && isWhitespace(value.charCodeAt(start))) start += 1; + while (end > start && isWhitespace(value.charCodeAt(end - 1))) end -= 1; + return value.slice(start, end); +} + // An inbound id flows into log fields AND response headers AND downstream -// subrequests, so anything multi-valued (Node joins dup headers as "v1, v2" -// or sometimes string[]) or containing CRLF/quotes/control chars must be -// dropped — pass-through would corrupt header framing or JSON escaping. +// subrequests. Header adapters may join repeated values with commas, so only +// the first token is considered; non-visible-ASCII and quote/backslash bytes +// are rejected before pass-through. // Dirty → mint fresh; preserving a maybe-poisoned upstream id defeats the // "single correlation token" goal. /** @@ -132,12 +144,11 @@ export function generateRequestId() { export function sanitizeRequestId(raw) { if (Array.isArray(raw)) raw = raw[0]; if (typeof raw !== "string") return null; - const first = raw.split(",")[0].trim(); + const first = trimAsciiWhitespace(raw.split(",")[0]); if (!first || first.length > 128) return null; - if (/[\s"\\]/.test(first)) return null; for (let i = 0; i < first.length; i++) { const code = first.charCodeAt(i); - if (code < 0x20 || code === 0x7f || (code >= 0x80 && code <= 0x9f)) return null; + if (code < 0x21 || code > 0x7e || code === 0x22 || code === 0x5c) return null; } return first; } @@ -220,18 +231,36 @@ export function recordRequestComplete({ */ export function formatError(err) { if (!err) return { error_message: "Unknown error" }; - if (err instanceof Error) { + if (isErrorObject(err)) { /** @type {Record} */ const out = { - error_name: err.name, - error_message: err.message, + error_name: errorStringField(err, "name") ?? "Error", + error_message: errorMessage(err), }; - const coded = /** @type {{ code?: unknown, reason?: unknown }} */ (err); - if (typeof coded.code === "string") out.error_code = coded.code; - else if (typeof coded.reason === "string") out.error_code = coded.reason; + const code = errorStringField(err, "code") ?? errorStringField(err, "reason"); + if (code !== null) out.error_code = code; return out; } - return { error_message: String(err) }; + return { error_message: errorMessage(err) }; +} + +/** @param {unknown} value @returns {value is Error} */ +function isErrorObject(value) { + try { + return value instanceof Error; + } catch { + return false; + } +} + +/** @param {Error} err @param {string} field */ +function errorStringField(err, field) { + try { + const value = /** @type {Record} */ (/** @type {unknown} */ (err))[field]; + return typeof value === "string" ? value : null; + } catch { + return null; + } } // Metrics bypass this gate entirely (in-memory registry, separate scrape diff --git a/shared/optimistic-retry.js b/shared/optimistic-retry.js new file mode 100644 index 0000000..bae79d5 --- /dev/null +++ b/shared/optimistic-retry.js @@ -0,0 +1,34 @@ +/** + * @template T + * @param {(attempt: number) => Promise} operation + * @param {{ + * attempts: number, + * isRetryableError?: (err: unknown) => boolean, + * onRetryableError?: (err: unknown, attempt: number) => void, + * shouldRetryResult?: (result: T, attempt: number) => boolean, + * onExhausted: () => T | Promise, + * }} options + * @returns {Promise} + */ +export async function withOptimisticRetries(operation, { + attempts, + isRetryableError = () => false, + onRetryableError, + shouldRetryResult, + onExhausted, +}) { + for (let attempt = 0; attempt < attempts; attempt += 1) { + try { + const result = await operation(attempt); + if (shouldRetryResult?.(result, attempt)) continue; + return result; + } catch (err) { + if (isRetryableError(err)) { + onRetryableError?.(err, attempt); + continue; + } + throw err; + } + } + return await onExhausted(); +} diff --git a/shared/owner-endpoint.js b/shared/owner-endpoint.js new file mode 100644 index 0000000..97071b6 --- /dev/null +++ b/shared/owner-endpoint.js @@ -0,0 +1,120 @@ +const OWNER_ENDPOINT_SERVICE_RE = { + "d1-runtime": /^d1-runtime(?:-[a-z0-9-]+)?$/, + "do-runtime": /^do-runtime(?:-[a-z0-9-]+)?$/, +}; + +const OWNER_ENDPOINT_HEADLESS_RE = { + "d1-runtime": /^d1-runtime-[0-9]+\.d1-runtime-headless(?:\.[a-z0-9-]+\.svc(?:\.cluster\.local)?)?$/, + "do-runtime": /^do-runtime-[0-9]+\.do-runtime-headless(?:\.[a-z0-9-]+\.svc(?:\.cluster\.local)?)?$/, +}; +const INVALID_OWNER_ENDPOINT_RE = /[/?#@[\]\\]/; +const IPV4_OCTET_RE = /^(?:0|[1-9]\d{0,2})$/; +const IntrinsicNumber = Number; +const IntrinsicString = String; +const IntrinsicURL = URL; +const intrinsicReflectApply = Reflect.apply; +const intrinsicRegExpTest = RegExp.prototype.test; +const intrinsicStringSplit = String.prototype.split; +const intrinsicUrlHashGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "hash")); +const intrinsicUrlHostGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "host")); +const intrinsicUrlHostnameGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "hostname")); +const intrinsicUrlPasswordGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "password")); +const intrinsicUrlPathnameGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "pathname")); +const intrinsicUrlPortGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "port")); +const intrinsicUrlSearchGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "search")); +const intrinsicUrlUsernameGet = /** @type {(this: URL) => string} */ (prototypeGetter(URL.prototype, "username")); + +/** @param {object | null} prototype @param {string} name */ +function prototypeGetter(prototype, name) { + let current = prototype; + while (current) { + const getter = Object.getOwnPropertyDescriptor(current, name)?.get; + if (getter) return getter; + current = Object.getPrototypeOf(current); + } + return undefined; +} + +/** @param {RegExp} regexp @param {string} value */ +function regexpTest(regexp, value) { + return intrinsicReflectApply(intrinsicRegExpTest, regexp, [value]); +} + +/** @param {string} value */ +function splitDots(value) { + return intrinsicReflectApply(intrinsicStringSplit, value, ["."]); +} + +/** @param {unknown} value */ +function numberValue(value) { + return intrinsicReflectApply(IntrinsicNumber, undefined, [value]); +} + +/** @param {unknown} value */ +function stringValue(value) { + return intrinsicReflectApply(IntrinsicString, undefined, [value]); +} + +/** @param {(this: URL) => string} getter @param {URL} url */ +function urlValue(getter, url) { + return intrinsicReflectApply(getter, url, []); +} + +/** + * @param {unknown} endpoint + * @param {number} port + * @param {"d1-runtime" | "do-runtime"} serviceName + * @returns {boolean} + */ +export function validOwnerEndpointForService(endpoint, port, serviceName) { + if (typeof endpoint !== "string" || !endpoint) return false; + const serviceRe = OWNER_ENDPOINT_SERVICE_RE[serviceName]; + const headlessRe = OWNER_ENDPOINT_HEADLESS_RE[serviceName]; + if (!serviceRe) throw new Error(`Unknown owner endpoint service ${serviceName}`); + if (regexpTest(INVALID_OWNER_ENDPOINT_RE, endpoint)) return false; + let url; + try { + url = new IntrinsicURL(`http://${endpoint}`); + } catch { + return false; + } + if ( + urlValue(intrinsicUrlHostGet, url) !== endpoint || + urlValue(intrinsicUrlUsernameGet, url) || + urlValue(intrinsicUrlPasswordGet, url) || + urlValue(intrinsicUrlPathnameGet, url) !== "/" || + urlValue(intrinsicUrlSearchGet, url) || + urlValue(intrinsicUrlHashGet, url) + ) { + return false; + } + if (urlValue(intrinsicUrlPortGet, url) !== stringValue(port)) return false; + const hostname = urlValue(intrinsicUrlHostnameGet, url); + return regexpTest(serviceRe, hostname) || + regexpTest(headlessRe, hostname) || + acceptablePrivateIpv4(hostname); +} + +/** + * @param {string} hostname + * @returns {boolean} + */ +function acceptablePrivateIpv4(hostname) { + const parts = splitDots(hostname); + if (parts.length !== 4) return false; + /** @type {number[]} */ + const octets = []; + for (let i = 0; i < parts.length; i++) { + const part = parts[i]; + if (!regexpTest(IPV4_OCTET_RE, part)) return false; + const value = numberValue(part); + if (value < 0 || value > 255) return false; + octets[i] = value; + } + const a = octets[0]; + const b = octets[1]; + return a === 10 || + (a === 100 && b >= 64 && b <= 127) || + (a === 172 && b >= 16 && b <= 31) || + (a === 192 && b === 168); +} diff --git a/shared/owner-forwarder.js b/shared/owner-forwarder.js index 60a90c1..63d6aa6 100644 --- a/shared/owner-forwarder.js +++ b/shared/owner-forwarder.js @@ -1,5 +1,6 @@ import { withInternalAuth } from "shared-internal-auth"; import { errorMessage } from "shared-errors"; +import { validOwnerEndpointForService } from "shared-owner-endpoint"; export const MAX_OWNER_FORWARD_HOPS = 2; @@ -35,6 +36,8 @@ function withRequestId(headers, requestId) { * @param {{ * env: Record, * endpoint?: string | null, + * endpointPort: number, + * endpointService: "d1-runtime" | "do-runtime", * pathname: string, * method?: string, * requestId?: string | null, @@ -49,6 +52,7 @@ function withRequestId(headers, requestId) { * buildHeaders(nextHopCount: number): HeadersInit, * logFields(): Record, * missingEndpointError(): Error, + * invalidEndpointError(): Error, * hopExhaustedError(): Error, * unavailableError(err: unknown): Error, * isTimeoutError?: (err: unknown) => boolean, @@ -58,6 +62,8 @@ function withRequestId(headers, requestId) { export async function forwardOwnerRequest({ env, endpoint, + endpointPort, + endpointService, pathname, method = "POST", requestId = null, @@ -72,11 +78,15 @@ export async function forwardOwnerRequest({ buildHeaders, logFields, missingEndpointError, + invalidEndpointError, hopExhaustedError, unavailableError, isTimeoutError = undefined, }) { if (!endpoint) throw missingEndpointError(); + if (!validOwnerEndpointForService(endpoint, endpointPort, endpointService)) { + throw invalidEndpointError(); + } const nextHopCount = hopCount + 1; if (nextHopCount > MAX_OWNER_FORWARD_HOPS) throw hopExhaustedError(); try { diff --git a/shared/owner-lease.js b/shared/owner-lease.js index d2c576f..7407011 100644 --- a/shared/owner-lease.js +++ b/shared/owner-lease.js @@ -1,4 +1,5 @@ import { envValueOr } from "shared-env"; +import { withOptimisticRetries } from "shared-optimistic-retry"; const utf8Decoder = new TextDecoder(); const OWNER_GENERATION_COUNTER = /^(?:0|[1-9]\d*)$/; @@ -44,7 +45,7 @@ export function parseOwnerRecord(raw) { } if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) return null; const record = /** @type {Record} */ (parsed); - if (!Number.isInteger(record.generation) || /** @type {number} */ (record.generation) < 0) { + if (!Number.isSafeInteger(record.generation) || /** @type {number} */ (record.generation) <= 0) { return null; } if ( @@ -127,7 +128,11 @@ export async function currentOwnerGenerationCounter(session, generationKey) { */ export async function nextOwnerGeneration(session, generationKey, currentGeneration = 0) { const currentCounter = await currentOwnerGenerationCounter(session, generationKey); - return Math.max(currentCounter + 1, currentGeneration + 1); + const previousGeneration = Math.max(currentCounter, currentGeneration); + if (!Number.isSafeInteger(previousGeneration) || previousGeneration >= Number.MAX_SAFE_INTEGER) { + throw new Error(`Owner generation counter is exhausted: ${generationKey}`); + } + return previousGeneration + 1; } /** @@ -164,16 +169,14 @@ export async function withOwnerWatchRetries(operation, { exhaustedMessage, }) { const maxAttempts = Number.isInteger(retries) && retries > 0 ? retries : 1; - for (let attempt = 0; attempt < maxAttempts; attempt += 1) { - try { - return await operation(); - } catch (err) { - if (typeof isWatchError === "function" && isWatchError(err)) { - if (attempt < maxAttempts - 1) continue; + return await withOptimisticRetries( + async () => await operation(), + { + attempts: maxAttempts, + isRetryableError: (err) => typeof isWatchError === "function" && isWatchError(err), + onExhausted: () => { throw createError(503, exhaustedCode, exhaustedMessage); - } - throw err; + }, } - } - throw createError(503, exhaustedCode, exhaustedMessage); + ); } diff --git a/shared/owner-protocol.js b/shared/owner-protocol.js index 2f4acc9..afc35fd 100644 --- a/shared/owner-protocol.js +++ b/shared/owner-protocol.js @@ -13,6 +13,7 @@ import { * exec(): Promise, * }} OwnerWriteMulti * @typedef {{ value: string | Uint8Array | ArrayBuffer | null | undefined, nowMs: number }} OwnerReadWithTimeResult + * @typedef {{ values: Array, nowMs: number }} OwnerSnapshotReadWithTimeResult */ /** * @template T @@ -21,6 +22,20 @@ import { * @returns {T | null} */ +/** + * @template T + * @param {string | Uint8Array | ArrayBuffer | null | undefined} value + * @param {number} nowMs + * @param {OwnerParser} parseOwner + * @returns {{ owner: T | null, nowMs: number }} + */ +function parseTimedOwnerRead(value, nowMs, parseOwner) { + if (!Number.isSafeInteger(nowMs) || nowMs < 0) { + throw new Error("Redis server time is invalid"); + } + return { owner: parseOwner(value), nowMs }; +} + /** * @param {string} prefix * @param {string} scope @@ -68,12 +83,22 @@ export async function readOwnerRecord(client, ownerKey, parseOwner) { */ export async function readOwnerRecordWithRedisTime(client, ownerKey, parseOwner) { const result = await client.getWithTime(ownerKey); - if (!Number.isSafeInteger(result.nowMs) || result.nowMs < 0) { - throw new Error("Redis server time is invalid"); - } + return parseTimedOwnerRead(result.value, result.nowMs, parseOwner); +} + +/** + * @template T + * @param {{ getManyWithTime(keys: string[]): Promise }} client + * @param {string} ownerKey + * @param {string[]} relatedKeys + * @param {OwnerParser} parseOwner + * @returns {Promise<{ owner: T | null, relatedValues: Array, nowMs: number }>} + */ +export async function readOwnerSnapshotWithRedisTime(client, ownerKey, relatedKeys, parseOwner) { + const result = await client.getManyWithTime([ownerKey, ...relatedKeys]); return { - owner: parseOwner(result.value), - nowMs: result.nowMs, + ...parseTimedOwnerRead(result.values[0], result.nowMs, parseOwner), + relatedValues: result.values.slice(1), }; } diff --git a/shared/queue-keys.js b/shared/queue-keys.js index 46df402..1c7ea3d 100644 --- a/shared/queue-keys.js +++ b/shared/queue-keys.js @@ -2,6 +2,8 @@ // corrupt parseStreamKey's anchor, so inline construction is a drift // risk — every tier imports from here. +import { isValidRouteNs, isValidRuntimeLoadNs, QUEUE_NAME_RE } from "./ns-pattern.js"; + /** * @typedef {{ ns: string, queue: string }} QueueKeyParts */ @@ -27,35 +29,42 @@ export const QUEUE_DELAYED_INDEX_KEY = "queue:index:delayed"; export function queueConsumerScanPrefix(ns) { return `queue-consumer:${ns}:`; } /** - * @lintignore kept as the JS mirror of Rust queue key parsers. + * @param {string} rest + * @param {(ns: string) => boolean} isValidNs + * @returns {QueueKeyParts | null} + */ +function parseQueueKeyRest(rest, isValidNs) { + const separator = rest.indexOf(":"); + if (separator <= 0) return null; + const ns = rest.slice(0, separator); + const queue = rest.slice(separator + 1); + if (!isValidNs(ns) || !QUEUE_NAME_RE.test(queue)) return null; + return { ns, queue }; +} + +/** * @param {string} key * @returns {QueueKeyParts | null} */ export function parseStreamKey(key) { - const m = key.match(/^queue:([^:]+):(.+):s$/); - return m ? { ns: m[1], queue: m[2] } : null; + if (!key.startsWith("queue:") || !key.endsWith(":s")) return null; + return parseQueueKeyRest(key.slice("queue:".length, -":s".length), isValidRuntimeLoadNs); } /** - * @lintignore kept as the JS mirror of Rust queue key parsers. * @param {string} key * @returns {QueueKeyParts | null} */ export function parseDelayedKey(key) { - const m = key.match(/^queue-delayed:([^:]+):(.+)$/); - return m ? { ns: m[1], queue: m[2] } : null; + if (!key.startsWith("queue-delayed:")) return null; + return parseQueueKeyRest(key.slice("queue-delayed:".length), isValidRuntimeLoadNs); } /** - * @lintignore kept as the JS mirror of Rust queue key parsers. * @param {string} key * @returns {QueueKeyParts | null} */ export function parseConsumerKey(key) { - const parts = key.split(":"); - if (parts.length < 3 || parts[0] !== "queue-consumer") return null; - const ns = parts[1]; - const queue = parts.slice(2).join(":"); - if (!ns || !queue) return null; - return { ns, queue }; + if (!key.startsWith("queue-consumer:")) return null; + return parseQueueKeyRest(key.slice("queue-consumer:".length), isValidRouteNs); } diff --git a/shared/redis-lock.js b/shared/redis-lock.js new file mode 100644 index 0000000..091a31a --- /dev/null +++ b/shared/redis-lock.js @@ -0,0 +1,59 @@ +import { randomHex } from "shared-random-id"; + +/** + * @typedef {{ key: string, token: string }} TokenLock + * @typedef {{ set(key: string, value: string, options: { nx?: boolean, ttl?: number, ifeq?: string }): Promise }} TokenLockSetClient + * @typedef {{ delIfEq(key: string, value: string): Promise }} TokenLockReleaseClient + */ + +/** @param {string} key @returns {TokenLock} */ +export function createTokenLock(key) { + return { key, token: randomHex(16) }; +} + +/** + * @param {TokenLockSetClient} client + * @param {TokenLock} lock + * @param {{ ttlSeconds: number }} options + */ +export async function acquireTokenLock(client, lock, { ttlSeconds }) { + if (!("set" in client) || typeof client.set !== "function") { + throw new TypeError("token lock acquire requires set()"); + } + return await client.set(lock.key, lock.token, { + nx: true, + ttl: ttlSeconds, + }) === "OK"; +} + +/** + * @param {TokenLockSetClient} client + * @param {TokenLock} lock + * @param {number} ttlSeconds + */ +export async function renewTokenLock(client, lock, ttlSeconds) { + return await client.set(lock.key, lock.token, { + ttl: ttlSeconds, + ifeq: lock.token, + }) === "OK"; +} + +/** + * Release is best-effort by contract: expiry bounds a leaked lock, while a + * release exception must not replace the operation's real success or failure. + * @param {TokenLockReleaseClient} client + * @param {TokenLock | null | undefined} lock + * @param {{ onError?: (err: unknown) => void }} [options] + */ +export async function releaseTokenLock(client, lock, { onError } = {}) { + if (!lock) return; + try { + await client.delIfEq(lock.key, lock.token); + } catch (err) { + try { + onError?.(err); + } catch { + // Logging is also best-effort on the release path. + } + } +} diff --git a/shared/redis-session.js b/shared/redis-session.js index 3d5de2f..91f201f 100644 --- a/shared/redis-session.js +++ b/shared/redis-session.js @@ -249,6 +249,11 @@ export class RedisSession { async incr(key) { return /** @type {number} */ (await this._exec("INCR", key)); } /** @param {string} key */ async get(key) { return decodeBulk(await this._exec("GET", key)); } + /** @param {string} key @param {RedisArg} value @param {RedisSetOptions} [opts] */ + async set(key, value, opts = {}) { + const reply = await this._exec(...buildSetArgs(key, value, opts)); + return reply === null ? null : decodeBulk(reply); + } /** @param {string[]} keys */ async getMany(keys) { const replies = /** @type {unknown[]} */ (await this._execPipeline( @@ -259,13 +264,22 @@ export class RedisSession { } /** @param {string} key */ async getWithTime(key) { - const [value, time] = await this._execPipeline("GET_TIME_PIPELINE", [ - ["GET", key], + const { values, nowMs } = await this.getManyWithTime([key]); + return { + value: values[0], + nowMs, + }; + } + /** @param {string[]} keys */ + async getManyWithTime(keys) { + if (keys.length === 0) throw new Error("getManyWithTime requires at least one key"); + const replies = await this._execPipeline("GET_TIME_PIPELINE", [ + ...keys.map((key) => /** @type {RedisCommand} */ (["GET", key])), ["TIME"], ]); return { - value: decodeBulk(value), - nowMs: decodeRedisTimeMs(time), + values: replies.slice(0, -1).map(decodeBulk), + nowMs: decodeRedisTimeMs(replies.at(-1)), }; } async time() { return decodeRedisTimeMs(await this._exec("TIME")); } diff --git a/shared/respond.js b/shared/respond.js index 8ae066f..b4aaabb 100644 --- a/shared/respond.js +++ b/shared/respond.js @@ -1,26 +1,31 @@ -// 101 upgrades can't be re-wrapped with the original body: workerd -// rejects a body on 101 (http.c++:1136-1149) and Response.clone() rejects -// WS handshakes (http.c++:1269). Rebuild with a null body + the hijacked -// WebSocket so the upgrade flows through and x-request-id still lands on -// the 101 headers. +// 101 upgrades can't be re-wrapped with the original body: workerd rejects a +// body on 101 and Response.clone() rejects WS handshakes. Rebuild with a null +// body plus the hijacked WebSocket so callers can safely replace headers. +/** + * @param {Response & { webSocket?: WebSocket | null }} response + * @param {HeadersInit} headers + * @returns {Response} + */ +export function rebuildResponseWithHeaders(response, headers) { + const init = /** @type {ResponseInit & { webSocket?: WebSocket }} */ ({ + status: response.status, + statusText: response.statusText, + headers, + }); + const webSocket = response.webSocket; + if (webSocket) init.webSocket = webSocket; + return new Response(response.status === 101 ? null : response.body, init); +} + /** * @param {Response & { webSocket?: WebSocket | null }} response * @param {string} requestId * @returns {Response} */ export function echoResponseWithRequestId(response, requestId) { - if (response.status === 101) { - const rebuilt = new Response(null, { - status: 101, - headers: response.headers, - webSocket: response.webSocket, - }); - rebuilt.headers.set("x-request-id", requestId); - return rebuilt; - } - const echoed = new Response(response.body, response); - echoed.headers.set("x-request-id", requestId); - return echoed; + const headers = new Headers(response.headers); + headers.set("x-request-id", requestId); + return rebuildResponseWithHeaders(response, headers); } /** @@ -29,7 +34,7 @@ export function echoResponseWithRequestId(response, requestId) { */ export async function discardResponseBody(response) { try { - await response.body?.cancel(); + void response.body?.cancel().catch(() => {}); } catch { // Best-effort cleanup only; the caller's status/error path owns behavior. } diff --git a/shared/s3-retry.js b/shared/s3-retry.js new file mode 100644 index 0000000..cd49e7d --- /dev/null +++ b/shared/s3-retry.js @@ -0,0 +1,46 @@ +import { discardResponseBody } from "./respond.js"; + +export const S3_TRANSIENT_RETRIES = 10; +const S3_RETRY_BASE_MS = 50; +const S3_RETRY_MAX_MS = 5_000; + +/** @param {number} ms */ +function delay(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** @param {number} attempt */ +function s3RetryDelayMs(attempt) { + return Math.min(S3_RETRY_BASE_MS * 2 ** attempt, S3_RETRY_MAX_MS); +} + +/** @param {Response} response */ +export function isTransientS3Response(response) { + return response.status === 429 || response.status >= 500; +} + +/** + * DeleteObjects is a POST, but deleting the same key set is safe to retry. + * Keep this scoped to S3 POST calls with known idempotent semantics. + * @param {{ fetch(url: string, init?: RequestInit): Promise }} client + * @param {string} url + * @param {RequestInit} init + */ +export async function fetchRetryableS3Post(client, url, init) { + for (let attempt = 0; attempt <= S3_TRANSIENT_RETRIES; attempt += 1) { + let response; + try { + response = await client.fetch(url, init); + } catch (err) { + if (attempt === S3_TRANSIENT_RETRIES) throw err; + await delay(Math.random() * s3RetryDelayMs(attempt)); + continue; + } + if (attempt === S3_TRANSIENT_RETRIES || !isTransientS3Response(response)) { + return response; + } + await discardResponseBody(response); + await delay(Math.random() * s3RetryDelayMs(attempt)); + } + throw new Error("unreachable S3 retry loop exit"); +} diff --git a/shared/secret-envelope.js b/shared/secret-envelope.js index d6291a2..070bcf7 100644 --- a/shared/secret-envelope.js +++ b/shared/secret-envelope.js @@ -12,7 +12,8 @@ export const SECRET_ENVELOPE_KID_ENV = "SECRET_ENVELOPE_KID"; const AES_GCM_TAG_BYTES = 16; const AES_GCM_IV_BYTES = 12; const AES_256_KEY_BYTES = 32; -const ENVELOPE_FIELDS = ["alg", "ct", "edek", "iv", "kid", "tag", "v"]; +const ENVELOPE_CANONICAL_FIELDS = ["v", "alg", "kid", "edek", "iv", "ct", "tag"]; +const ENVELOPE_FIELDS = ENVELOPE_CANONICAL_FIELDS.toSorted(); const utf8Encoder = new TextEncoder(); const utf8FatalDecoder = new TextDecoder("utf-8", { fatal: true }); @@ -52,6 +53,14 @@ function bytesToText(bytes) { export { bytesToBase64 }; +/** @param {Record} envelope */ +function canonicalEnvelopeJson(envelope) { + /** @type {Record} */ + const ordered = {}; + for (const field of ENVELOPE_CANONICAL_FIELDS) ordered[field] = envelope[field]; + return JSON.stringify(ordered); +} + /** * @param {unknown} value * @param {string} [fieldName] @@ -218,7 +227,7 @@ export async function encryptSecretValue(plaintext, { env, hashKey, fieldName, r edekBytes.set(wrappedDek.tag, dekIv.length + wrappedDek.ct.length); const payload = await aesGcmEncrypt(dek, payloadIv, textBytes(plaintext), payloadAadBytes(hashKey, fieldName)); - return `${SECRET_ENVELOPE_PREFIX}${JSON.stringify({ + return `${SECRET_ENVELOPE_PREFIX}${canonicalEnvelopeJson({ v: SECRET_ENVELOPE_VERSION, alg: SECRET_ENVELOPE_ALG, kid, @@ -252,7 +261,7 @@ function parseEnvelope(value) { if (fields.length !== ENVELOPE_FIELDS.length || fields.some((field, idx) => field !== ENVELOPE_FIELDS[idx])) { throw new SecretEnvelopeError("invalid_envelope", "secret envelope has unknown or missing fields"); } - if (JSON.stringify(parsed) !== json) { + if (canonicalEnvelopeJson(envelope) !== json) { throw new SecretEnvelopeError("invalid_envelope", "secret envelope JSON is not canonical"); } if (envelope.v !== SECRET_ENVELOPE_VERSION || envelope.alg !== SECRET_ENVELOPE_ALG) { diff --git a/shared/task-identity.js b/shared/task-identity.js index 3afce15..272960a 100644 --- a/shared/task-identity.js +++ b/shared/task-identity.js @@ -15,6 +15,7 @@ const DEFAULT_IDENTITY_TIMEOUT_MS = 1000; * serviceLabel: string, * unavailableCode: string, * createError: ErrorFactory, + * validateEndpoint?: (endpoint: string, port: number) => boolean, * }} TaskIdentityResolverOptions * @typedef {Record} EnvLike */ @@ -55,6 +56,7 @@ export function createTaskIdentityResolver({ serviceLabel, unavailableCode, createError, + validateEndpoint = undefined, }) { /** @type {TaskIdentity | null} */ let cachedIdentity = null; @@ -69,13 +71,12 @@ export function createTaskIdentityResolver({ return createError(503, unavailableCode, message); } - /** - * @param {EnvLike} env - * @returns {number} - */ - function taskPort(env) { - const raw = Number(envValueOr(env[`${envPrefix}_TASK_PORT`], defaultPort)); - return Number.isFinite(raw) && raw > 0 ? raw : defaultPort; + /** @param {TaskIdentity} identity */ + function requireValidEndpoint(identity) { + if (validateEndpoint?.(identity.endpoint, defaultPort) === false) { + throw error(`${serviceLabel} task endpoint is invalid`); + } + return identity; } /** @@ -140,7 +141,7 @@ export function createTaskIdentityResolver({ if (!taskId || !endpoint) { throw error(`${envPrefix}_TASK_ID and ${envPrefix}_TASK_ENDPOINT must be configured together`); } - return { taskId, endpoint, source: "env" }; + return requireValidEndpoint({ taskId, endpoint, source: "env" }); } /** @@ -154,11 +155,11 @@ export function createTaskIdentityResolver({ if (!taskId) throw error("ECS task metadata did not include TaskARN"); const privateIpv4 = firstPrivateIpv4(metadata, env); if (!privateIpv4) throw error("ECS task metadata did not include a private IPv4 address"); - return { + return requireValidEndpoint({ taskId, - endpoint: `${privateIpv4}:${taskPort(env)}`, + endpoint: `${privateIpv4}:${defaultPort}`, source: "ecs-metadata", - }; + }); } /** diff --git a/shared/vendor/aws-sigv4.js b/shared/vendor/aws-sigv4.js index 6dde753..a654ee5 100644 --- a/shared/vendor/aws-sigv4.js +++ b/shared/vendor/aws-sigv4.js @@ -31,31 +31,57 @@ var DEFAULT_UNSIGNABLE_HEADERS = /* @__PURE__ */ new Set([ "user-agent", "x-amzn-trace-id" ]); -var RFC3986_EXTRA_ESCAPE_RE = /[!'()*]/g; -var LOWER_HEX = "0123456789abcdef"; var AWS_ALGORITHM = "AWS4-HMAC-SHA256"; var AWS_REQUEST = "aws4_request"; var UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD"; -var CONTROL_CHAR_RE = /[\u0000-\u001f\u007f]/u; -var WHITESPACE_RE = /\s/u; -var AUTH_PARAM_SEPARATOR_RE = /[,=;]/u; -var HTTP_METHOD_RE = /^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/u; -var ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})$/u; -var SIGNING_DATE_ERROR = "signingDate must be a valid Date, ISO-8601 string, or YYYYMMDDTHHMMSSZ string"; -var IDEMPOTENT_METHODS = /* @__PURE__ */ new Set(["GET", "HEAD", "OPTIONS", "PUT", "DELETE"]); +var LOWER_HEX = "0123456789abcdef"; +var inFlightSigningKeys = /* @__PURE__ */ new WeakMap(); async function signatureHex(options) { - const secretAccessKeyHash = options.secretAccessKeyHash ?? await sha256Hex(options.secretAccessKey); - const cacheKey = ["sigv4", secretAccessKeyHash, options.date, options.region, options.service].join(","); - let signingKey = options.cache?.get(cacheKey); - if (!signingKey) { - const kDate = await hmac(`AWS4${options.secretAccessKey}`, options.date); - const kRegion = await hmac(kDate, options.region); - const kService = await hmac(kRegion, options.service); - signingKey = await hmac(kService, AWS_REQUEST); - options.cache?.set(cacheKey, signingKey); + let signingKey; + if (options.cache === void 0) { + signingKey = await deriveSigningKey(options); + } else { + const secretAccessKeyHash = options.secretAccessKeyHash ?? await sha256Hex(options.secretAccessKey); + const cacheKey = ["sigv4", secretAccessKeyHash, options.date, options.region, options.service].join(","); + const cachedSigningKey = options.cache.get(cacheKey); + signingKey = isSigningKeyCacheMiss(cachedSigningKey) ? await deriveCachedSigningKey(options, options.cache, cacheKey) : cachedSigningKey; } return hex(await hmac(signingKey, options.stringToSign)); } +function isSigningKeyCacheMiss(value) { + return value === void 0 || value === null; +} +async function deriveCachedSigningKey(options, cache, cacheKey) { + let byKey = inFlightSigningKeys.get(cache); + if (byKey === void 0) { + byKey = /* @__PURE__ */ new Map(); + inFlightSigningKeys.set(cache, byKey); + } + const existing = byKey.get(cacheKey); + if (existing !== void 0) { + return existing; + } + const derivation = (async () => { + try { + const signingKey = await deriveSigningKey(options); + cache.set(cacheKey, signingKey); + return signingKey; + } finally { + byKey.delete(cacheKey); + if (byKey.size === 0) { + inFlightSigningKeys.delete(cache); + } + } + })(); + byKey.set(cacheKey, derivation); + return derivation; +} +async function deriveSigningKey(options) { + const kDate = await hmac(`AWS4${options.secretAccessKey}`, options.date); + const kRegion = await hmac(kDate, options.region); + const kService = await hmac(kRegion, options.service); + return hmac(kService, AWS_REQUEST); +} async function sha256Hex(value) { const bytes = cryptoBufferSource(value); return hex(await crypto.subtle.digest("SHA-256", bytes)); @@ -88,7 +114,30 @@ function hex(input) { function hexNibble(value) { return LOWER_HEX.charAt(value); } -async function prepareBody(body, headers, materialize) { +async function prepareSigningBody(body, headers, options) { + if (options.unsignedPayload && !headers.has(AMZ_CONTENT_SHA256_HEADER)) { + headers.set(AMZ_CONTENT_SHA256_HEADER, UNSIGNED_PAYLOAD); + } + const hasBody = body !== null && body !== void 0; + const computePayloadHash = !headers.has(AMZ_CONTENT_SHA256_HEADER); + const materialize = options.replay && shouldMaterializeBodyForReplay(body) || computePayloadHash && hasBody; + const prepared = await prepareBody(body, headers, materialize, options.signal); + let payloadHash = headers.get(AMZ_CONTENT_SHA256_HEADER); + if (payloadHash === null) { + if (prepared.bytes === void 0) { + throw new Error("body bytes must be materialized before payload hashing"); + } + payloadHash = await sha256Hex(prepared.bytes); + options.signal?.throwIfAborted(); + if (hasBody || options.service === "s3") { + headers.set(AMZ_CONTENT_SHA256_HEADER, payloadHash); + } + } + return { body: prepared.body, payloadHash }; +} +async function prepareBody(body, headers, materialize, signal) { + signal?.throwIfAborted(); + assertSupportedBody(body); if (body === null || body === void 0) { return { body, bytes: new Uint8Array() }; } @@ -99,20 +148,29 @@ async function prepareBody(body, headers, materialize) { if (contentType) { headers.set(CONTENT_TYPE_HEADER, contentType); } - const bytes2 = new Uint8Array(await request.arrayBuffer()); + const bytes2 = signal && request.body ? await readStreamBytes(request.body, signal) : new Uint8Array(await request.arrayBuffer()); + signal?.throwIfAborted(); return { body: bytes2, bytes: bytes2 }; } setGeneratedContentType(body, headers); - if (!materialize) { - return { body, bytes: new Uint8Array() }; - } if (body instanceof ReadableStream) { - const bytes2 = new Uint8Array(await new Response(body).arrayBuffer()); + assertReadableStreamUsable(body); + if (!materialize) { + return { body }; + } + const bytes2 = await readStreamBytes(body, signal); return { body: bytes2, bytes: bytes2 }; } - const bytes = await bodyBytes(body); + if (!materialize && (typeof body === "string" || body instanceof Blob)) { + return { body }; + } + const bytes = await bodyBytes(body, signal); + signal?.throwIfAborted(); return { body: stableMaterializedBody(body, bytes), bytes }; } +function shouldMaterializeBodyForReplay(body) { + return body !== void 0 && body !== null && typeof body !== "string" && !(body instanceof Blob); +} function rejectManualFormDataContentType(headers) { if (headers.has(CONTENT_TYPE_HEADER)) { throw new TypeError("FormData content-type must be generated by the runtime"); @@ -126,44 +184,107 @@ function setGeneratedContentType(body, headers) { headers.set(CONTENT_TYPE_HEADER, "text/plain;charset=UTF-8"); } else if (body instanceof URLSearchParams) { headers.set(CONTENT_TYPE_HEADER, "application/x-www-form-urlencoded;charset=UTF-8"); - } else if (body instanceof Blob && body.type) { - headers.set(CONTENT_TYPE_HEADER, body.type); + } else if (body instanceof Blob) { + if (body.type) { + headers.set(CONTENT_TYPE_HEADER, body.type); + } } } -async function bodyBytes(body) { +async function bodyBytes(body, signal) { if (typeof body === "string") { return textEncoder.encode(body); } if (body instanceof Uint8Array) { - return body; + return new Uint8Array(body); } if (body instanceof ArrayBuffer) { - return new Uint8Array(body); + return new Uint8Array(body).slice(); } if (ArrayBuffer.isView(body)) { - return new Uint8Array(body.buffer, body.byteOffset, body.byteLength); + return new Uint8Array(body.buffer, body.byteOffset, body.byteLength).slice(); } if (body instanceof URLSearchParams) { return textEncoder.encode(body.toString()); } if (body instanceof Blob) { + if (signal) { + return readStreamBytes(body.stream(), signal); + } return new Uint8Array(await body.arrayBuffer()); } throw new TypeError("body must be a string, Blob, URLSearchParams, ArrayBuffer, or ArrayBufferView"); } -function shouldHashPayload(body, headers, unsignedPayload) { - return body !== void 0 && body !== null && !unsignedPayload && !headers.has(AMZ_CONTENT_SHA256_HEADER); +function assertReadableStreamUsable(body) { + try { + void new Response(body); + } catch (error) { + if (error instanceof TypeError) { + throw new TypeError("ReadableStream body must not be disturbed or locked", { cause: error }); + } + throw error; + } +} +async function readStreamBytes(stream, signal) { + const reader = stream.getReader(); + const chunks = []; + let byteLength = 0; + const onAbort = () => { + if (signal) { + void reader.cancel(signal.reason).catch(() => void 0); + } + }; + signal?.addEventListener("abort", onAbort, { once: true }); + try { + for (; ; ) { + signal?.throwIfAborted(); + const result = await reader.read(); + signal?.throwIfAborted(); + if (result.done) { + break; + } + const value = result.value; + if (!(value instanceof Uint8Array)) { + throw new TypeError("ReadableStream body must yield Uint8Array chunks"); + } + const chunk = new Uint8Array(value); + chunks.push(chunk); + byteLength += chunk.byteLength; + } + } catch (err) { + if (signal?.aborted) { + throw signal.reason; + } + void reader.cancel(err).catch(() => void 0); + throw err; + } finally { + signal?.removeEventListener("abort", onAbort); + reader.releaseLock(); + } + const bytes = new Uint8Array(byteLength); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.byteLength; + } + return bytes; +} +function isAsyncIterable(value) { + return typeof value[Symbol.asyncIterator] === "function"; } -async function prepareHashedBody(body, headers, unsignedPayload, materialize = false) { - const hashPayload = shouldHashPayload(body, headers, unsignedPayload); - const preparedBody = await prepareBody(body, headers, materialize || hashPayload); - if (hashPayload) { - headers.set(AMZ_CONTENT_SHA256_HEADER, await sha256Hex(preparedBody.bytes)); +function assertSupportedBody(body) { + if (body === null || body === void 0) { + return; + } + if (typeof body === "string" || body instanceof Blob || body instanceof FormData || body instanceof ReadableStream || body instanceof URLSearchParams || body instanceof ArrayBuffer || ArrayBuffer.isView(body)) { + return; } - return preparedBody; + if (isAsyncIterable(body)) { + throw new TypeError("async iterable bodies are not supported; use a ReadableStream"); + } + throw new TypeError("body must be a string, Blob, URLSearchParams, ArrayBuffer, or ArrayBufferView"); } function stableMaterializedBody(body, bytes) { - if (typeof body === "string" || body instanceof Uint8Array) { + if (typeof body === "string") { return body; } if (bytes.buffer instanceof ArrayBuffer) { @@ -191,6 +312,7 @@ function canonicalHeaderBlock(url, headers, options) { }; } function validateSignedHeaderValues(headers, options) { + rejectMandatoryHeaderExclusions(headers, options); const overwrittenHeaderNames = new Set((options.overwrittenHeaderNames || []).map((value) => value.toLowerCase())); for (const header of signedHeaderNames(headers, options)) { if (header !== HOST_HEADER && !overwrittenHeaderNames.has(header)) { @@ -204,7 +326,7 @@ function signerOverwrittenHeaderNames(hasSessionToken) { function signedHeaderNames(headers, options) { const userUnsignable = new Set((options.unsignableHeaders || []).map((value) => value.toLowerCase())); return [.../* @__PURE__ */ new Set([HOST_HEADER, ...headers.keys()])].filter((header) => header !== AUTHORIZATION_HEADER).filter((header) => { - if (MANDATORY_SIGNED_HEADERS.has(header)) { + if (isMandatorySignedHeader(header, options.service)) { return true; } if (userUnsignable.has(header)) { @@ -213,6 +335,18 @@ function signedHeaderNames(headers, options) { return options.signAllHeaders || !DEFAULT_UNSIGNABLE_HEADERS.has(header); }).sort(); } +function rejectMandatoryHeaderExclusions(headers, options) { + for (const value of options.unsignableHeaders || []) { + const header = value.toLowerCase(); + const isPresentDynamicHeader = headers.has(header) && (header.startsWith("x-amz-") || options.service === "s3" && header === "content-md5"); + if (MANDATORY_SIGNED_HEADERS.has(header) || isPresentDynamicHeader) { + throw new TypeError(`unsignableHeaders must not include mandatory signed header ${header}`); + } + } +} +function isMandatorySignedHeader(header, service) { + return MANDATORY_SIGNED_HEADERS.has(header) || header.startsWith("x-amz-") || service === "s3" && header === "content-md5"; +} function canonicalHeaderValue(value, name) { rejectNonPrintableAsciiHeaderValue(value, name); return value.trim().replace(/\s+/gu, " "); @@ -220,6 +354,8 @@ function canonicalHeaderValue(value, name) { function rejectNonPrintableAsciiHeaderValue(value, name) { rejectNonPrintableAscii(value, `${name} header value must contain only printable ASCII characters`); } +var ISO_DATE_RE = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})$/u; +var SIGNING_DATE_ERROR = "signingDate must be a valid Date, ISO-8601 string, or YYYYMMDDTHHMMSSZ string"; function optionalAmzDate(value) { if (value === void 0 || value === null) { return void 0; @@ -240,25 +376,15 @@ function formatAmzDate(value) { throw new TypeError(SIGNING_DATE_ERROR); } const date = typeof value === "string" ? new Date(value) : value; - if (!(date instanceof Date)) { - throw new TypeError(SIGNING_DATE_ERROR); - } - if (Number.isNaN(dateTimeValue(date))) { + if (!(date instanceof Date) || Number.isNaN(date.getTime())) { throw new TypeError(SIGNING_DATE_ERROR); } - const amzDate = Date.prototype.toISOString.call(date).replace(/[:-]|\.\d{3}/g, ""); + const amzDate = date.toISOString().replace(/[:-]|\.\d{3}/g, ""); if (!/^\d{8}T\d{6}Z$/u.test(amzDate) || !isValidCompactAmzDate(amzDate)) { throw new TypeError(SIGNING_DATE_ERROR); } return amzDate; } -function dateTimeValue(date) { - try { - return Date.prototype.getTime.call(date); - } catch { - throw new TypeError(SIGNING_DATE_ERROR); - } -} function isValidIsoDate(value) { const match = ISO_DATE_RE.exec(value); if (!match) { @@ -283,6 +409,10 @@ function isValidDateParts(year, month, day, hour, minute, second) { date.setUTCFullYear(year); return date.getUTCFullYear() === year && date.getUTCMonth() === month - 1 && date.getUTCDate() === day && date.getUTCHours() === hour && date.getUTCMinutes() === minute && date.getUTCSeconds() === second; } +var AUTH_PARAM_SEPARATOR_RE = /[,=;]/u; +var CONTROL_CHAR_RE = /[\u0000-\u001f\u007f]/u; +var HEADER_NAME_RE = /^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/u; +var WHITESPACE_RE = /\s/u; var CLIENT_SIGNING_OPTION_KEYS = /* @__PURE__ */ new Set([ "service", "region", @@ -293,6 +423,13 @@ var CLIENT_SIGNING_OPTION_KEYS = /* @__PURE__ */ new Set([ "doubleUrlEncode" ]); var UNSIGNABLE_HEADER_SNAPSHOTS = /* @__PURE__ */ new WeakMap(); +function snapshotSignAwsRequestOptions(value) { + requireOptionsObject(value, "signAwsRequest options are required"); + const snapshot = { ...value }; + validateCredentialOptions(snapshot, "signAwsRequest options are required"); + snapshot.unsignableHeaders = snapshotUnsignableHeaders(value, snapshot.unsignableHeaders, "unsignableHeaders"); + return snapshot; +} function normalizeClientSigningOptions(options) { if (options === void 0) { return {}; @@ -300,28 +437,21 @@ function normalizeClientSigningOptions(options) { if (options === null || typeof options !== "object") { throw new TypeError("init.signing must be an object"); } - for (const key of Object.keys(options)) { + const source = { ...options }; + for (const key of Object.keys(source)) { if (!CLIENT_SIGNING_OPTION_KEYS.has(key)) { throw new TypeError(`${signingOptionDisplayName(key)} cannot override client credentials or transport options`); } } - const record = options; - const normalized = { ...record }; - const service = optionalCredentialComponent(ownNonNullOption(record, "service"), "init.signing.service"); - setOrDelete(normalized, "service", service); - const region = optionalCredentialComponent(ownNonNullOption(record, "region"), "init.signing.region"); - setOrDelete(normalized, "region", region); - const signingDate = optionalAmzDate(ownNonNullOption(record, "signingDate")); - setOrDelete(normalized, "signingDate", signingDate); - const unsignedPayload = optionalBoolean(ownOption(record, "unsignedPayload"), "init.signing.unsignedPayload"); - const signAllHeaders = optionalBoolean(ownOption(record, "signAllHeaders"), "init.signing.signAllHeaders"); - const doubleUrlEncode = optionalBoolean(ownOption(record, "doubleUrlEncode"), "init.signing.doubleUrlEncode"); - const unsignableHeaders = snapshotUnsignableHeaders(options, ownOption(record, "unsignableHeaders"), "init.signing.unsignableHeaders"); - setOrDelete(normalized, "unsignedPayload", unsignedPayload); - setOrDelete(normalized, "signAllHeaders", signAllHeaders); - setOrDelete(normalized, "unsignableHeaders", unsignableHeaders); - setOrDelete(normalized, "doubleUrlEncode", doubleUrlEncode); - return normalized; + return { + service: optionalCredentialComponent(nullAsUndefined(source.service), "init.signing.service"), + region: optionalCredentialComponent(nullAsUndefined(source.region), "init.signing.region"), + signingDate: optionalAmzDate(nullAsUndefined(source.signingDate)), + unsignedPayload: optionalBoolean(source.unsignedPayload, "init.signing.unsignedPayload"), + signAllHeaders: optionalBoolean(source.signAllHeaders, "init.signing.signAllHeaders"), + unsignableHeaders: snapshotUnsignableHeaders(options, source.unsignableHeaders, "init.signing.unsignableHeaders"), + doubleUrlEncode: optionalBoolean(source.doubleUrlEncode, "init.signing.doubleUrlEncode") + }; } function signingOptionDisplayName(key) { if (key.length === 0) { @@ -340,8 +470,8 @@ function validateCredentialOptions(options, message) { const record = options; requireCredentialComponent(record.accessKeyId, "accessKeyId"); requireSecretAccessKey(record.secretAccessKey); - requireCredentialComponent(record.service, "service"); - requireCredentialComponent(record.region, "region"); + requireLowercaseCredentialComponent(record.service, "service"); + requireLowercaseCredentialComponent(record.region, "region"); if (record.sessionToken !== void 0) { validateSessionToken(record.sessionToken); } @@ -365,10 +495,19 @@ function requireCredentialComponent(value, name) { function requireSecretAccessKey(value) { requireString(value, "secretAccessKey"); rejectControlChars(value, "secretAccessKey"); + if (!value.isWellFormed()) { + throw new TypeError("secretAccessKey must contain well-formed UTF-16"); + } +} +function requireLowercaseCredentialComponent(value, name) { + requireCredentialComponent(value, name); + if (value !== value.toLowerCase()) { + throw new TypeError(`${name} must be lowercase`); + } } function requireNonNegativeInteger(value, name) { - if (!Number.isInteger(value) || value < 0) { - throw new TypeError(`${name} must be a non-negative integer`); + if (!Number.isSafeInteger(value) || value < 0) { + throw new TypeError(`${name} must be a non-negative integer within the safe integer range`); } return value; } @@ -390,6 +529,9 @@ function optionalBoolean(value, name) { function resolveUnsignedPayload(explicit, service) { return explicit ?? service === "s3"; } +function resolveDoubleUrlEncode(explicit, service) { + return explicit ?? service !== "s3"; +} function normalizeUnsignableHeaders(value, name) { if (value === void 0) { return void 0; @@ -401,6 +543,9 @@ function normalizeUnsignableHeaders(value, name) { if (typeof header !== "string" || header.length === 0) { throw new TypeError(`${name} must contain only non-empty strings`); } + if (!HEADER_NAME_RE.test(header)) { + throw new TypeError(`${name} must contain only valid header names`); + } return header; }); } @@ -408,14 +553,11 @@ function snapshotUnsignableHeaders(owner, source, name) { if (source === void 0) { return void 0; } - if (!isIterable(source)) { - return normalizeUnsignableHeaders(source, name); - } - if (!isOneShotIterable(source)) { + if (!isIterable(source) || !isOneShotIterable(source)) { return normalizeUnsignableHeaders(source, name); } const cached = UNSIGNABLE_HEADER_SNAPSHOTS.get(owner); - if (cached && cached.source === source) { + if (cached?.source === source) { if (cached.result.ok) { return cached.result.value; } @@ -474,32 +616,21 @@ function rejectSurroundingWhitespace(value, name) { throw new TypeError(`${name} must not contain leading or trailing whitespace`); } } -function ownNonNullOption(record, name) { - const value = ownOption(record, name); - return value === null ? void 0 : value; -} -function ownOption(record, name) { - return Object.hasOwn(record, name) ? record[name] : void 0; -} function optionalCredentialComponent(value, name) { if (value === void 0) { return void 0; } - requireCredentialComponent(value, name); + requireLowercaseCredentialComponent(value, name); return value; } -function setOrDelete(record, key, value) { - if (value === void 0) { - Reflect.deleteProperty(record, key); - } else { - record[key] = value; - } +function nullAsUndefined(value) { + return value === null ? void 0 : value; } function isOneShotIterable(value) { return Object.is(value[Symbol.iterator](), value); } function isIterable(value) { - return value !== null && typeof value !== "string" && typeof value[Symbol.iterator] === "function"; + return value !== null && (typeof value === "object" || typeof value === "function") && typeof value[Symbol.iterator] === "function"; } function isSigningCache(value) { if (value === null || typeof value !== "object" || value instanceof WeakMap) { @@ -508,10 +639,11 @@ function isSigningCache(value) { const candidate = value; return typeof candidate.get === "function" && typeof candidate.set === "function"; } +var RFC3986_EXTRA_ESCAPE_RE = /[!'()*]/g; function parseRequestUrl(input) { const raw = String(input); - if (typeof input === "string" && /\s/u.test(raw)) { - throw new TypeError("url must not contain unescaped whitespace"); + if (typeof input === "string" && /[\s\u0000-\u001f\u007f]/u.test(raw)) { + throw new TypeError("url must not contain unescaped whitespace or control characters"); } if (typeof input === "string" && raw.includes("\\")) { throw new TypeError("url must not contain backslashes"); @@ -576,7 +708,7 @@ function canonicalQuery(search) { const separator = part.indexOf("="); const key = separator === -1 ? part : part.slice(0, separator); const value = separator === -1 ? "" : part.slice(separator + 1); - return [canonicalQueryComponent(key), canonicalQueryComponent(value)]; + return [canonicalUriComponent(key), canonicalUriComponent(value)]; }).sort(([ak, av], [bk, bv]) => compareCodepoint(ak, bk) || compareCodepoint(av, bv)).map(([key, value]) => `${key}=${value}`).join("&"); } function hasDotPathSegment(pathname) { @@ -592,9 +724,6 @@ function stripUrlFragment(value) { function hasMalformedPercentEncoding(value) { return /%(?![0-9A-Fa-f]{2})/u.test(value); } -function canonicalQueryComponent(value) { - return canonicalUriComponent(value); -} function compareCodepoint(left, right) { if (left < right) { return -1; @@ -642,7 +771,30 @@ function canonicalSingleEncodedPathname(pathname) { return out; } function canonicalDoubleEncodedPathname(pathname) { - return strictEncode(pathname).replace(/%2F/gu, "/"); + return strictEncode(wireEncodedPathname(pathname)).replace(/%2F/gu, "/"); +} +function wireEncodedPathname(pathname) { + let out = ""; + for (let index = 0; index < pathname.length; ) { + const char = pathname[index]; + if (char === "/" || char === "%" && isHexPair(pathname, index + 1)) { + const width = char === "/" ? 1 : 3; + out += pathname.slice(index, index + width); + index += width; + continue; + } + const codePoint = pathname.codePointAt(index); + if (codePoint === void 0) { + break; + } + const charValue = String.fromCodePoint(codePoint); + out += shouldWhatwgEncodePathCodePoint(codePoint) ? strictEncode(charValue) : charValue; + index += charValue.length; + } + return out; +} +function shouldWhatwgEncodePathCodePoint(codePoint) { + return codePoint > 126 || codePoint === 34 || codePoint === 60 || codePoint === 62 || codePoint === 94 || codePoint === 96 || codePoint === 123 || codePoint === 125; } function collapsePathSlashes(pathname) { return pathname.replace(/\/+/gu, "/"); @@ -674,9 +826,63 @@ function isHexPair(value, index) { function isUnreservedByte(byte) { return byte >= 65 && byte <= 90 || byte >= 97 && byte <= 122 || byte >= 48 && byte <= 57 || byte === 45 || byte === 46 || byte === 95 || byte === 126; } +var HTTP_METHOD_RE = /^[!#$%&'*+.^_`|~0-9A-Za-z-]+$/u; +var IDEMPOTENT_METHODS = /* @__PURE__ */ new Set(["GET", "HEAD", "OPTIONS", "PUT", "DELETE"]); +function resolveClientSignRequest(input, init) { + return resolveClientRequest(input, init).request; +} +function resolveClientFetchRequest(input, init) { + const { request, inheritedRedirect, explicitRedirect } = resolveClientRequest(input, init); + const redirectPolicy = resolveFetchRedirectPolicy(inheritedRedirect, explicitRedirect); + request.init.redirect = "manual"; + return { ...request, redirectPolicy }; +} +function resolveClientRequest(input, init) { + const inputSnapshot = input instanceof Request ? snapshotRequestInput(input) : void 0; + const requestUrl = parseRequestUrl(inputSnapshot?.url ?? input); + const initSnapshot = snapshotRequestInit(init); + const requestInit = inputSnapshot ? mergeDefinedRequestInit(inputSnapshot.init, initSnapshot) : initSnapshot; + const signal = requestInit.signal ?? void 0; + signal?.throwIfAborted(); + rejectNoCorsMode(requestInit.mode); + const headers = inputSnapshot === void 0 ? new Headers(requestInit.headers) : mergeHeaders(inputSnapshot.headers, requestInit.headers); + let body = requestInit.body; + let method = requestInit.method; + if (inputSnapshot !== void 0) { + rejectUsedRequestBody(inputSnapshot.bodyUsed, body); + if ((body === void 0 || body === null) && inputSnapshot.body) { + body = inputSnapshot.body; + } + if (method === void 0) { + method = inputSnapshot.method; + } + } + const normalizedMethod = normalizeMethod(method === void 0 ? defaultMethod(body) : method); + rejectFetchForbiddenMethod(normalizedMethod); + rejectRequestBodyForGetHead(normalizedMethod, body); + requestInit.method = normalizedMethod; + requestInit.headers = headers; + if (body === void 0) { + delete requestInit.body; + } else { + requestInit.body = body; + } + return { + request: { + requestUrl, + init: requestInit, + method: normalizedMethod, + headers, + body, + signal + }, + inheritedRedirect: inputSnapshot?.init.redirect, + explicitRedirect: initSnapshot.redirect + }; +} function mergeHeaders(base, override) { const headers = new Headers(base); - if (override) { + if (override !== void 0) { new Headers(override).forEach((value, name) => { headers.set(name, value); }); @@ -692,12 +898,26 @@ function mergeDefinedRequestInit(base, override) { } return out; } +function snapshotRequestInit(value) { + if (value !== void 0 && value !== null && typeof value !== "object") { + throw new TypeError("init must be an object"); + } + if (value === void 0 || value === null) { + return {}; + } + return { ...value }; +} function rejectEmptyHeader(headers, name) { if (headers.get(name) === "") { throw new TypeError(`${name} must not be empty`); } } -function requestInitFromRequest(request) { +function snapshotRequestInput(request) { + const url = request.url; + const method = request.method; + const headers = new Headers(request.headers); + const body = request.body; + const bodyUsed = request.bodyUsed; const init = { cache: request.cache, credentials: request.credentials, @@ -712,24 +932,21 @@ function requestInitFromRequest(request) { }; const duplex = request.duplex; if (duplex === "half") { - return { ...init, duplex }; - } - return init; -} -function requestBodyForInput(input, init) { - if (init.body !== void 0) { - return init.body; + return { + url, + method, + headers, + body, + bodyUsed, + init: { ...init, duplex } + }; } - return input instanceof Request ? input.body : void 0; + return { url, method, headers, body, bodyUsed, init }; } -function methodForRequest(input, init) { - if (init.method !== void 0) { - return normalizeMethod(init.method); +function rejectUsedRequestBody(bodyUsed, override) { + if ((override === void 0 || override === null) && bodyUsed) { + throw new TypeError("Request body has already been used"); } - if (input instanceof Request) { - return normalizeMethod(input.method); - } - return defaultMethod(init.body); } function defaultMethod(body) { return hasRequestBody(body) ? "POST" : "GET"; @@ -740,6 +957,11 @@ function normalizeMethod(method) { } return method.toUpperCase(); } +function rejectFetchForbiddenMethod(method) { + if (method === "CONNECT" || method === "TRACE" || method === "TRACK") { + throw new TypeError(`SigV4Client cannot use Fetch-forbidden method ${method}`); + } +} function isIdempotentMethod(method) { return IDEMPOTENT_METHODS.has(method); } @@ -751,9 +973,6 @@ function rejectRequestBodyForGetHead(method, body) { function hasRequestBody(body) { return body !== void 0 && body !== null; } -function assertRequestCanRepresentSignedUrl(url, service) { - assertParsedRequestCanRepresentSignedUrl(parseRequestUrl(url).pathname, service); -} function assertParsedRequestCanRepresentSignedUrl(pathname, service) { if (hasDotPathSegment(pathname)) { throw new TypeError(`SigV4Client cannot represent ${service} URLs with dot segments; use signAwsRequest`); @@ -770,18 +989,92 @@ function requestInitForSignedRequest(base, signed) { } return out; } +function createSignedRequest(url, init, expectedHeaders) { + let request; + try { + request = new Request(url, init); + } catch (err) { + const duplex = init.duplex; + if (!(err instanceof TypeError) || !(init.body instanceof ReadableStream) || duplex !== void 0) { + throw err; + } + request = new Request(url, { + ...init, + duplex: "half" + }); + } + assertSignedRequestHeadersPreserved(expectedHeaders, request.headers); + return request; +} +function rejectNoCorsMode(mode) { + if (mode === "no-cors") { + throw new TypeError('SigV4Client cannot sign requests with mode "no-cors" because required headers may be removed'); + } +} +function validateRequestBeforeTransport(request) { + try { + request.signal.throwIfAborted(); + rejectNoCorsMode(request.mode); + if (request.redirect !== "manual") { + throw new TypeError('SigV4Client.fetch signed Request must use redirect: "manual"'); + } + return request.signal; + } catch (err) { + cancelRequestBody(request, err); + throw err; + } +} +function isRedirectResponse(response) { + return response.type === "opaqueredirect" || response.redirected || response.status === 301 || response.status === 302 || response.status === 303 || response.status === 307 || response.status === 308; +} +function assertSignedRequestHeadersPreserved(expected, actual) { + const authorization = expected.get(AUTHORIZATION_HEADER); + if (authorization === null || actual.get(AUTHORIZATION_HEADER) !== authorization) { + throw new TypeError("runtime removed or rewrote the authorization header after signing"); + } + const signedHeaders = /(?:^|,\s*)SignedHeaders=([^,\s]+)/u.exec(authorization)?.[1]; + if (!signedHeaders) { + throw new Error("generated SigV4 authorization is missing SignedHeaders"); + } + for (const name of signedHeaders.split(";")) { + if (name === HOST_HEADER) { + continue; + } + if (actual.get(name) !== expected.get(name)) { + throw new TypeError(`runtime removed or rewrote the signed ${name} header`); + } + } +} function bindFetch(fetchFn) { return Object.is(fetchFn, globalThis.fetch) ? fetchFn.bind(globalThis) : fetchFn; } -function isAbortError(err, request) { - return request.signal.aborted || err instanceof DOMException && err.name === "AbortError"; +function resolveFetchRedirectPolicy(inherited, explicit) { + if (explicit !== void 0) { + if (explicit === "follow") { + throw new TypeError('SigV4Client.fetch does not allow redirect: "follow"; redirected requests must be re-signed'); + } + if (explicit !== "error" && explicit !== "manual") { + throw new TypeError('redirect must be "error" or "manual"'); + } + return explicit; + } + return inherited === "manual" ? "manual" : "error"; } -function sleep(ms, signal) { - if (!signal) { - return new Promise((resolve) => setTimeout(resolve, ms)); +function cancelRequestBody(request, reason) { + try { + const cancellation = request.body?.cancel(reason); + if (cancellation !== void 0) { + void cancellation.catch(() => void 0); + } + } catch { } +} +function isAbortError(err) { + return err instanceof DOMException && err.name === "AbortError"; +} +function sleep(ms, signal) { if (signal.aborted) { - return Promise.reject(abortReason(signal)); + return Promise.reject(signal.reason); } return new Promise((resolve, reject) => { let timeout; @@ -790,7 +1083,7 @@ function sleep(ms, signal) { clearTimeout(timeout); } signal.removeEventListener("abort", onAbort); - reject(abortReason(signal)); + reject(signal.reason); }; signal.addEventListener("abort", onAbort, { once: true }); timeout = setTimeout(() => { @@ -799,55 +1092,61 @@ function sleep(ms, signal) { }, ms); }); } -function abortReason(signal) { - return signal.reason ?? new DOMException("aborted", "AbortError"); -} -async function cancelResponseBody(response) { +function cancelResponseBody(response) { try { - await response.body?.cancel(); + const cancellation = response.body?.cancel(); + if (cancellation) { + void cancellation.catch(() => void 0); + } } catch { } } async function signAwsRequest(options) { - return signAwsRequestInternal(options); -} -async function signAwsRequestInternal(options, secretAccessKeyHash, parsedRequestUrl) { + const snapshot = snapshotSignAwsRequestOptions(options); + const signal = snapshot.signal ?? void 0; + signal?.throwIfAborted(); + const signed = await signAwsRequestInternal(snapshot); + signal?.throwIfAborted(); + return signed; +} +async function signAwsRequestInternal(options, secretAccessKeyHash, parsedRequestUrl, reusablePreparedBody) { validateCredentialOptions(options, "signAwsRequest options are required"); const cache = requireSigningCache(options.cache, "cache"); requireDefinedOption(options.url, "url"); const requestUrl = parsedRequestUrl ?? parseRequestUrl(options.url); const url = requestUrl.url; const method = normalizeMethod(options.method === void 0 ? defaultMethod(options.body) : options.method); - const headers = new Headers(options.headers || {}); + const headers = new Headers(options.headers); rejectEmptyHeader(headers, AMZ_CONTENT_SHA256_HEADER); const unsignedPayload = resolveUnsignedPayload(optionalBoolean(options.unsignedPayload, "unsignedPayload"), options.service); const signAllHeaders = optionalBoolean(options.signAllHeaders, "signAllHeaders"); - const unsignableHeaders = snapshotUnsignableHeaders(options, options.unsignableHeaders, "unsignableHeaders"); - const doubleUrlEncode = optionalBoolean(options.doubleUrlEncode, "doubleUrlEncode") ?? false; - if (unsignedPayload && !headers.has(AMZ_CONTENT_SHA256_HEADER)) { - headers.set(AMZ_CONTENT_SHA256_HEADER, UNSIGNED_PAYLOAD); - } + const unsignableHeaders = normalizeUnsignableHeaders(options.unsignableHeaders, "unsignableHeaders"); + const doubleUrlEncode = resolveDoubleUrlEncode(optionalBoolean(options.doubleUrlEncode, "doubleUrlEncode"), options.service); const explicitAmzDate = optionalAmzDate(options.signingDate); headers.set(HOST_HEADER, url.host); if (options.sessionToken) { headers.set(AMZ_SECURITY_TOKEN_HEADER, options.sessionToken); } validateSignedHeaderValues(headers, { + service: options.service, signAllHeaders, unsignableHeaders, overwrittenHeaderNames: signerOverwrittenHeaderNames(options.sessionToken !== void 0) }); const canonicalPath = canonicalPathname(requestUrl.pathname, options.service, doubleUrlEncode); - const preparedBody = await prepareHashedBody(options.body, headers, unsignedPayload); + const preparedBody = reusablePreparedBody ?? await prepareSigningBody(options.body, headers, { + service: options.service, + unsignedPayload, + replay: false, + signal: options.signal ?? void 0 + }); const amzDate = explicitAmzDate ?? formatAmzDate(/* @__PURE__ */ new Date()); const date = amzDate.slice(0, 8); const credentialScope = `${date}/${options.region}/${options.service}/${AWS_REQUEST}`; headers.set(AMZ_DATE_HEADER, amzDate); - const canonicalPayloadHash = await canonicalPayloadHashValue(headers, preparedBody.bytes); - if (options.service === "s3" && !headers.has(AMZ_CONTENT_SHA256_HEADER)) { - headers.set(AMZ_CONTENT_SHA256_HEADER, canonicalPayloadHash); - } + const canonicalPayloadHash = preparedBody.payloadHash; const { canonicalHeaders, signedHeaders } = canonicalHeaderBlock(url, headers, { + service: options.service, signAllHeaders, unsignableHeaders }); @@ -861,9 +1160,10 @@ async function signAwsRequestInternal(options, secretAccessKeyHash, parsedReques canonicalPayloadHash ].join("\n"); const stringToSign = [AWS_ALGORITHM, amzDate, credentialScope, await sha256Hex(canonicalRequest)].join("\n"); + const resolvedSecretAccessKeyHash = typeof secretAccessKeyHash === "function" ? await secretAccessKeyHash() : secretAccessKeyHash; const signature = await signatureHex({ secretAccessKey: options.secretAccessKey, - secretAccessKeyHash, + secretAccessKeyHash: resolvedSecretAccessKeyHash, date, region: options.region, service: options.service, @@ -882,209 +1182,184 @@ async function signAwsRequestInternal(options, secretAccessKeyHash, parsedReques body: preparedBody.body }; } -async function canonicalPayloadHashValue(headers, body) { - const explicit = headers.get(AMZ_CONTENT_SHA256_HEADER); - if (explicit) { - return explicit; - } - return sha256Hex(body); -} var MAX_RETRY_DELAY_MS = 2147483647; var SigV4Client = class { - accessKeyId; - secretAccessKey; - secretAccessKeyHash; - sessionToken; - service; - region; - cache; - retries; - initialRetryDelayMs; - maxRetryDelayMs; - unsignedPayload; - signAllHeaders; - unsignableHeaders; - doubleUrlEncode; - fetchFn; + #accessKeyId; + #secretAccessKey; + #secretAccessKeyHash; + #sessionToken; + #service; + #region; + #cache; + #retries; + #initialRetryDelayMs; + #maxRetryDelayMs; + #unsignedPayload; + #signAllHeaders; + #unsignableHeaders; + #doubleUrlEncode; + #fetchFn; constructor(options) { validateCredentialOptions(options, "SigV4Client options are required"); - this.accessKeyId = options.accessKeyId; - this.secretAccessKey = options.secretAccessKey; - this.sessionToken = options.sessionToken; - this.service = options.service; - this.region = options.region; - this.cache = requireSigningCache(options.cache, "cache") ?? /* @__PURE__ */ new Map(); - this.retries = requireNonNegativeInteger(options.retries ?? 0, "retries"); - this.initialRetryDelayMs = requireNonNegativeFiniteNumber(options.initialRetryDelayMs ?? 50, "initialRetryDelayMs"); - this.maxRetryDelayMs = requireNonNegativeFiniteNumber(options.maxRetryDelayMs ?? 5e3, "maxRetryDelayMs"); - this.unsignedPayload = optionalBoolean(options.unsignedPayload, "unsignedPayload"); - this.signAllHeaders = optionalBoolean(options.signAllHeaders, "signAllHeaders"); - this.unsignableHeaders = normalizeUnsignableHeaders(options.unsignableHeaders, "unsignableHeaders"); - this.doubleUrlEncode = optionalBoolean(options.doubleUrlEncode, "doubleUrlEncode"); + this.#accessKeyId = options.accessKeyId; + this.#secretAccessKey = options.secretAccessKey; + this.#sessionToken = options.sessionToken; + this.#service = options.service; + this.#region = options.region; + this.#cache = requireSigningCache(options.cache, "cache") ?? /* @__PURE__ */ new Map(); + this.#retries = requireNonNegativeInteger(options.retries === void 0 ? 0 : options.retries, "retries"); + this.#initialRetryDelayMs = requireNonNegativeFiniteNumber(options.initialRetryDelayMs === void 0 ? 50 : options.initialRetryDelayMs, "initialRetryDelayMs"); + this.#maxRetryDelayMs = requireNonNegativeFiniteNumber(options.maxRetryDelayMs === void 0 ? 5e3 : options.maxRetryDelayMs, "maxRetryDelayMs"); + this.#unsignedPayload = optionalBoolean(options.unsignedPayload, "unsignedPayload"); + this.#signAllHeaders = optionalBoolean(options.signAllHeaders, "signAllHeaders"); + this.#unsignableHeaders = normalizeUnsignableHeaders(options.unsignableHeaders, "unsignableHeaders"); + this.#doubleUrlEncode = optionalBoolean(options.doubleUrlEncode, "doubleUrlEncode"); const fetchFn = options.fetch === void 0 ? globalThis.fetch : options.fetch; if (typeof fetchFn !== "function") { throw new TypeError(options.fetch === void 0 ? "fetch is not available" : "fetch must be a function"); } - this.fetchFn = bindFetch(fetchFn); - } - async sign(input, init = {}) { - const requestInit = input instanceof Request ? mergeDefinedRequestInit(requestInitFromRequest(input), init) : { ...init }; - const signingOptions = normalizeClientSigningOptions(requestInit.signing); - delete requestInit.signing; - let url; - let method = requestInit.method; - let headers = requestInit.headers; - let body = requestInit.body; - if (input instanceof Request) { - url = input.url; - if (method === void 0) { - method = input.method; - } - headers = mergeHeaders(input.headers, headers); - if (body === void 0 && input.body) { - body = input.clone().body; - } - } else { - url = input; - } - const normalizedMethod = normalizeMethod(method === void 0 ? defaultMethod(body) : method); - rejectRequestBodyForGetHead(normalizedMethod, body); - const service = signingOptions.service ?? this.service; - const requestUrl = parseRequestUrl(url); - assertParsedRequestCanRepresentSignedUrl(requestUrl.pathname, service); - const signed = await signAwsRequestInternal({ - accessKeyId: this.accessKeyId, - secretAccessKey: this.secretAccessKey, - sessionToken: this.sessionToken, - service, - region: signingOptions.region ?? this.region, - cache: this.cache, - unsignedPayload: signingOptions.unsignedPayload ?? this.unsignedPayload, - signAllHeaders: signingOptions.signAllHeaders ?? this.signAllHeaders, - unsignableHeaders: signingOptions.unsignableHeaders ?? this.unsignableHeaders, - doubleUrlEncode: signingOptions.doubleUrlEncode ?? this.doubleUrlEncode, - signingDate: signingOptions.signingDate, - method: normalizedMethod, - url, - headers, - body - }, await this.getSecretAccessKeyHash(), requestUrl); - assertRequestCanRepresentSignedUrl(signed.url, service); - const signedInit = requestInitForSignedRequest(requestInit, signed); - try { - return new Request(signed.url, signedInit); - } catch (err) { - if (err instanceof TypeError) { - return new Request(signed.url, { - ...signedInit, - duplex: "half" - }); - } - throw err; - } + this.#fetchFn = bindFetch(fetchFn); } - async fetch(input, init = {}) { - const requestInit = { - ...init, - signing: normalizeClientSigningOptions(init.signing) - }; - const method = methodForRequest(input, requestInit); - rejectRequestBodyForGetHead(method, requestBodyForInput(input, requestInit)); - const service = requestInit.signing.service ?? this.service; - assertRequestCanRepresentSignedUrl(input instanceof Request ? input.url : input, service); - const replayBody = this.retries > 0 && isIdempotentMethod(method); - const retryInit = await reusableRequestInitForInput(input, requestInit, { - defaultService: this.service, - defaultUnsignedPayload: this.unsignedPayload, - defaultSignAllHeaders: this.signAllHeaders, - defaultUnsignableHeaders: this.unsignableHeaders, - hasClientSessionToken: this.sessionToken !== void 0, - replayBody - }); - for (let attempt = 0; attempt <= this.retries; attempt += 1) { - const fetchFn = this.fetchFn; - const request = await this.sign(input, retryInit); + async sign(input, init) { + const request = resolveClientSignRequest(input, init); + const signing = this.#resolveSigningOptions(request.init.signing); + delete request.init.signing; + assertParsedRequestCanRepresentSignedUrl(request.requestUrl.pathname, signing.service); + return this.#signResolvedRequest(request, signing); + } + async #signResolvedRequest(request, signing, preparedBody) { + request.signal?.throwIfAborted(); + const signed = await signAwsRequestInternal({ + accessKeyId: this.#accessKeyId, + secretAccessKey: this.#secretAccessKey, + sessionToken: this.#sessionToken, + service: signing.service, + region: signing.region, + cache: this.#cache, + unsignedPayload: signing.unsignedPayload, + signAllHeaders: signing.signAllHeaders, + unsignableHeaders: signing.unsignableHeaders, + doubleUrlEncode: signing.doubleUrlEncode, + signingDate: signing.signingDate, + signal: request.signal, + method: request.method, + url: request.requestUrl.href, + headers: request.headers, + body: request.body + }, () => this.#getSecretAccessKeyHash(), request.requestUrl, preparedBody); + request.signal?.throwIfAborted(); + const signedInit = requestInitForSignedRequest(request.init, signed); + return createSignedRequest(signed.url, signedInit, signed.headers); + } + async fetch(input, init) { + const request = resolveClientFetchRequest(input, init); + const signing = this.#resolveSigningOptions(request.init.signing); + delete request.init.signing; + assertParsedRequestCanRepresentSignedUrl(request.requestUrl.pathname, signing.service); + const retryableMethod = isIdempotentMethod(request.method); + const prepared = await prepareFetchRequest(request, signing, this.#sessionToken !== void 0, this.#retries > 0 && retryableMethod); + const fetchFn = this.#fetchFn; + for (let attempt = 0; attempt <= this.#retries; attempt += 1) { + const signedRequest = await this.#signResolvedRequest(prepared.request, signing, prepared.preparedBody); + const attemptSignal = validateRequestBeforeTransport(signedRequest); let response; try { - response = await fetchFn(request); + response = await fetchFn(signedRequest); } catch (err) { - if (attempt === this.retries || !isIdempotentMethod(request.method) || isAbortError(err, request)) { + attemptSignal.throwIfAborted(); + if (attempt === this.#retries || !retryableMethod || isAbortError(err)) { throw err; } - await sleep(Math.random() * this.retryDelayMs(attempt), request.signal); + await sleep(Math.random() * this.#retryDelayMs(attempt), attemptSignal); continue; } - const retryableResponse = isIdempotentMethod(request.method) && (response.status >= 500 || response.status === 429); - if (attempt === this.retries || !retryableResponse) { - return response; + if (attemptSignal.aborted) { + cancelResponseBody(response); + attemptSignal.throwIfAborted(); } - await cancelResponseBody(response); - if (request.signal.aborted) { - throw abortReason(request.signal); + if (prepared.request.redirectPolicy === "manual" && response.redirected) { + cancelResponseBody(response); + attemptSignal.throwIfAborted(); + throw new TypeError('SigV4Client.fetch custom transport followed a redirect despite redirect: "manual"'); } - await sleep(Math.random() * this.retryDelayMs(attempt), request.signal); + if (prepared.request.redirectPolicy === "error" && isRedirectResponse(response)) { + cancelResponseBody(response); + attemptSignal.throwIfAborted(); + throw new TypeError("SigV4Client.fetch received a redirect response; redirect targets must be re-signed"); + } + const retryableResponse = retryableMethod && (response.status >= 500 || response.status === 429); + if (attempt === this.#retries || !retryableResponse) { + return response; + } + cancelResponseBody(response); + attemptSignal.throwIfAborted(); + await sleep(Math.random() * this.#retryDelayMs(attempt), attemptSignal); } throw new Error("unreachable retry loop exit"); } - getSecretAccessKeyHash() { - if (this.secretAccessKeyHash !== void 0) { - return this.secretAccessKeyHash; + #resolveSigningOptions(value) { + const options = normalizeClientSigningOptions(value); + const service = options.service ?? this.#service; + return { + service, + region: options.region ?? this.#region, + unsignedPayload: resolveUnsignedPayload(options.unsignedPayload ?? this.#unsignedPayload, service), + signAllHeaders: options.signAllHeaders ?? this.#signAllHeaders, + unsignableHeaders: options.unsignableHeaders ?? this.#unsignableHeaders, + doubleUrlEncode: resolveDoubleUrlEncode(options.doubleUrlEncode ?? this.#doubleUrlEncode, service), + signingDate: options.signingDate + }; + } + #getSecretAccessKeyHash() { + if (this.#secretAccessKeyHash !== void 0) { + return this.#secretAccessKeyHash; } - const hash = sha256Hex(this.secretAccessKey).catch((err) => { - if (this.secretAccessKeyHash === hash) { - this.secretAccessKeyHash = void 0; + const hash = sha256Hex(this.#secretAccessKey).catch((err) => { + if (this.#secretAccessKeyHash === hash) { + this.#secretAccessKeyHash = void 0; } throw err; }); - this.secretAccessKeyHash = hash; + this.#secretAccessKeyHash = hash; return hash; } - retryDelayMs(attempt) { - return Math.min(MAX_RETRY_DELAY_MS, this.maxRetryDelayMs, this.initialRetryDelayMs * 2 ** attempt); + #retryDelayMs(attempt) { + return Math.min(MAX_RETRY_DELAY_MS, this.#maxRetryDelayMs, this.#initialRetryDelayMs * 2 ** attempt); } }; -async function reusableRequestInitForInput(input, init, options) { - if (!(input instanceof Request)) { - return reusableRequestInit(init, options); - } - const inputInit = { - ...init, - method: init.method ?? input.method, - headers: mergeHeaders(input.headers, init.headers) - }; - if (init.body === void 0 && input.body) { - inputInit.body = input.clone().body; - } - return reusableRequestInit(inputInit, options); -} -async function reusableRequestInit(init, options) { - const headers = new Headers(init.headers || {}); +async function prepareFetchRequest(request, signing, hasSessionToken, replay) { + const headers = new Headers(request.headers); rejectEmptyHeader(headers, AMZ_CONTENT_SHA256_HEADER); - const service = init.signing?.service ?? options.defaultService; - const unsignedPayload = resolveUnsignedPayload(init.signing?.unsignedPayload ?? options.defaultUnsignedPayload, service); - const unsignableHeaders = init.signing?.unsignableHeaders ?? options.defaultUnsignableHeaders; validateSignedHeaderValues(headers, { - signAllHeaders: init.signing?.signAllHeaders ?? options.defaultSignAllHeaders, - unsignableHeaders, - overwrittenHeaderNames: signerOverwrittenHeaderNames(options.hasClientSessionToken) + service: signing.service, + signAllHeaders: signing.signAllHeaders, + unsignableHeaders: signing.unsignableHeaders, + overwrittenHeaderNames: signerOverwrittenHeaderNames(hasSessionToken) + }); + const preparedBody = await prepareSigningBody(request.body, headers, { + service: signing.service, + unsignedPayload: signing.unsignedPayload, + replay, + signal: request.signal }); - const materializeBody = options.replayBody && (init.body instanceof FormData || init.body instanceof ReadableStream); - const hashPayload = shouldHashPayload(init.body, headers, unsignedPayload); - if (!materializeBody && !hashPayload) { - return { - ...init, - headers - }; - } - const body = await prepareHashedBody(init.body, headers, unsignedPayload, materializeBody); const out = { - ...init, + ...request.init, headers }; - if (body.body !== void 0) { - out.body = body.body; + if (preparedBody.body === void 0) { + delete out.body; + } else { + out.body = preparedBody.body; } - return out; + return { + request: { + ...request, + init: out, + headers, + body: preparedBody.body + }, + preparedBody + }; } export { SigV4Client, diff --git a/shared/version.js b/shared/version.js index 4f7e8ff..4b34de9 100644 --- a/shared/version.js +++ b/shared/version.js @@ -10,7 +10,7 @@ const VERSION_RE = /^v([1-9][0-9]*)$/; * @returns {string} */ export function formatVersion(n) { - if (typeof n !== "number" || !Number.isInteger(n) || n < 1) { + if (typeof n !== "number" || !Number.isSafeInteger(n) || n < 1) { throw new Error(`invalid version number ${n}`); } return `v${n}`; @@ -24,7 +24,9 @@ export function formatVersion(n) { export function parseVersion(tag) { if (typeof tag !== "string") return null; const m = tag.match(VERSION_RE); - return m ? Number.parseInt(m[1], 10) : null; + if (!m) return null; + const parsed = Number.parseInt(m[1], 10); + return Number.isSafeInteger(parsed) ? parsed : null; } // `v:` infix separates the integer-indexed bundle namespace from sibling @@ -42,6 +44,12 @@ export function bundleKey(ns, name, version) { return `worker:${ns}:${name}:v:${n}`; } +// Monotonic immutable-version allocator for one logical worker. +/** @param {string} ns @param {string} name */ +export function nextVersionKey(ns, name) { + return `worker:${ns}:${name}:next_version`; +} + // Active-route hash for a namespace: field=workerName, value=`v`. Control // is the sole writer; centralized here so cross-tier readers cannot drift from // the key grammar (reader set in docs/redis-key-layout.md). @@ -57,6 +65,37 @@ export function patternsKey(host) { return `patterns:${host}`; } +export const NAMESPACES_KEY = "namespaces"; +export const DECLARED_HOSTS_KEY = "declared-hosts"; +export const ROUTES_CHANNEL = "routes:invalidate"; +export const ROUTES_FLUSH_CHANNEL = "routes:flush"; +export const PATTERNS_CHANNEL = "patterns:invalidate"; +const HOSTS_PREFIX = "hosts:"; +const NS_HOSTS_PREFIX = "ns-hosts:"; +const HOST_DECLARATIONS_PREFIX = "host-declarations:"; +export const HOSTS_SCAN_PATTERN = `${HOSTS_PREFIX}*`; +export const HOST_DECLARATIONS_SCAN_PATTERN = `${HOST_DECLARATIONS_PREFIX}*`; + +/** @param {string} ns */ +export function hostsKey(ns) { + return `${HOSTS_PREFIX}${ns}`; +} + +/** @param {string} key */ +export function namespaceFromHostsKey(key) { + return key.startsWith(HOSTS_PREFIX) ? key.slice(HOSTS_PREFIX.length) : ""; +} + +/** @param {string} ns */ +export function nsHostsKey(ns) { + return `${NS_HOSTS_PREFIX}${ns}`; +} + +/** @param {string} host */ +export function hostDeclarationsKey(host) { + return `${HOST_DECLARATIONS_PREFIX}${host}`; +} + // Retained-version ZSET for a worker: score=int version, member=`v`. /** @param {string} ns @param {string} worker @returns {string} */ export function workerVersionsKey(ns, worker) { @@ -69,3 +108,47 @@ export function workerVersionsKey(ns, worker) { export function doStorageIdKey(ns, worker) { return `worker:do-storage:${ns}:${worker}`; } + +// DO runtime owns the records; Control uses the storage-scoped pattern during +// whole-worker cleanup. Keep both sides on the same encoded Redis prefix. +export const DO_OWNER_SCOPE_PREFIX = "do:owner:scope:"; + +/** @param {string} storageId */ +export function doOwnerScopeScanPatternForStorage(storageId) { + return `${DO_OWNER_SCOPE_PREFIX}${encodeURIComponent(`${storageId}:`)}*`; +} + +// Per-worker lifecycle lock. Control owns acquisition/release; other tiers may +// WATCH it when creating state that whole-worker delete must discover. +export const WHOLE_DELETE_LOCK_KIND = "whole"; +export const VERSION_DELETE_LOCK_KIND = "version"; + +/** @param {string} ns @param {string} worker */ +export function deleteLockKey(ns, worker) { + return `worker-delete-lock:${ns}:${worker}`; +} + +/** + * @param {"whole" | "version"} kind + * @param {string} token + */ +export function formatDeleteLockToken(kind, token) { + if ( + (kind !== WHOLE_DELETE_LOCK_KIND && kind !== VERSION_DELETE_LOCK_KIND) || + typeof token !== "string" || !token + ) { + throw new TypeError("invalid worker delete lock token"); + } + return `${kind}:${token}`; +} + +/** @param {unknown} value @returns {"whole" | "version" | null} */ +export function parseDeleteLockKind(value) { + if (typeof value !== "string") return null; + const separator = value.indexOf(":"); + if (separator <= 0 || separator === value.length - 1) return null; + const kind = value.slice(0, separator); + return kind === WHOLE_DELETE_LOCK_KIND || kind === VERSION_DELETE_LOCK_KIND + ? kind + : null; +} diff --git a/shared/workerd-compat-flags.js b/shared/workerd-compat-flags.js index 33e72b9..b06e252 100644 --- a/shared/workerd-compat-flags.js +++ b/shared/workerd-compat-flags.js @@ -1,13 +1,25 @@ /* - * Mirrors workerd v1.20260701.1 src/workerd/io/compatibility-date.capnp. - * Regenerate on every workerd pin bump from an upstream workerd source checkout: + * The experimental flags mirror workerd v1.20260701.1 + * src/workerd/io/compatibility-date.capnp. Refresh them on every workerd pin + * bump from an upstream source checkout: * * node scripts/extract-workerd-experimental-compat-flags.mjs \ * /path/to/workerd/src/workerd/io/compatibility-date.capnp + * + * The dynamic-worker minimum/default dates and required error-serialization + * behavior below are WDL policy rather than upstream mirror data. */ export const WORKERD_EXPERIMENTAL_COMPAT_FLAGS_SOURCE_VERSION = "1.20260701.1"; +// WDL supports one forward-only dynamic-worker compatibility surface. Static +// platform workers keep their independently pinned workerd service dates. +export const MIN_DYNAMIC_WORKER_COMPATIBILITY_DATE = "2026-04-01"; +export const DEFAULT_DYNAMIC_WORKER_COMPATIBILITY_DATE = "2026-04-24"; +export const ENHANCED_ERROR_SERIALIZATION_DEFAULT_DATE = "2026-04-21"; +export const ENHANCED_ERROR_SERIALIZATION_FLAG = "enhanced_error_serialization"; +export const LEGACY_ERROR_SERIALIZATION_FLAG = "legacy_error_serialization"; + export const WORKERD_EXPERIMENTAL_COMPAT_FLAGS = Object.freeze([ "allow_insecure_inefficient_logged_eval", "allow_irrevocable_stub_storage", diff --git a/system-workers/s3-cleanup/src/index.js b/system-workers/s3-cleanup/src/index.js index a75fd8e..2e3a765 100644 --- a/system-workers/s3-cleanup/src/index.js +++ b/system-workers/s3-cleanup/src/index.js @@ -11,7 +11,6 @@ import { createLogLevelBinder, logStructured as emitStructuredLog, } from "../../../shared/observability.js"; -import { discardResponseBody } from "../../../shared/respond.js"; import { errorMessage } from "../../../shared/errors.js"; import { bytesToBase64 } from "../../../shared/base64.js"; import { @@ -29,6 +28,10 @@ import { xmlUnescape, } from "../../../shared/s3-xml.js"; import { encodeS3Query } from "../../../shared/s3-query.js"; +import { + S3_TRANSIENT_RETRIES, + fetchRetryableS3Post, +} from "../../../shared/s3-retry.js"; const MAX_ATTEMPTS = 10; const BACKOFF_MAX_MS = 30 * 60_000; @@ -37,9 +40,6 @@ const CRON_BATCH = 100; const MAX_DELETE_PAGES_PER_RUN = 1; const MAX_LIST_PAGES = 1000; const PROCESSING_LEASE_MS = 30 * 60_000; -const S3_TRANSIENT_RETRIES = 10; -const S3_RETRY_BASE_MS = 50; -const S3_RETRY_MAX_MS = 5_000; const DB_BINDING = "S3_CLEANUP_DB"; const SERVICE = "s3-cleanup"; const utf8Encoder = new TextEncoder(); @@ -59,47 +59,6 @@ export function nextBackoffMs(attempts) { return Math.min(BACKOFF_BASE_MS * 2 ** (attempts - 1), BACKOFF_MAX_MS); } -/** @param {number} ms */ -function delay(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -/** @param {number} attempt */ -function s3RetryDelayMs(attempt) { - return Math.min(S3_RETRY_BASE_MS * 2 ** attempt, S3_RETRY_MAX_MS); -} - -/** @param {Response} response */ -function isTransientS3Response(response) { - return response.status === 429 || response.status >= 500; -} - -/** - * DeleteObjects is a POST, but deleting the same key set is safe to retry. - * Keep this scoped to S3 POST calls with known idempotent semantics. - * @param {SigV4Client} client - * @param {string} url - * @param {RequestInit} init - */ -async function fetchRetryableS3Post(client, url, init) { - for (let attempt = 0; attempt <= S3_TRANSIENT_RETRIES; attempt += 1) { - let response; - try { - response = await client.fetch(url, init); - } catch (err) { - if (attempt === S3_TRANSIENT_RETRIES) throw err; - await delay(Math.random() * s3RetryDelayMs(attempt)); - continue; - } - if (attempt === S3_TRANSIENT_RETRIES || !isTransientS3Response(response)) { - return response; - } - await discardResponseBody(response); - await delay(Math.random() * s3RetryDelayMs(attempt)); - } - throw new Error("unreachable S3 retry loop exit"); -} - /** * @param {"debug" | "info" | "warn" | "error"} level * @param {string} event diff --git a/terraform/README.md b/terraform/README.md index 9447031..259d41f 100644 --- a/terraform/README.md +++ b/terraform/README.md @@ -411,9 +411,8 @@ Manager secrets. ### Test Hooks -`d1_test_hooks_enabled` and `do_test_hooks_enabled` default to `false` and are -guarded to test-named compute stacks. Keep both disabled for normal live-ready -service state. +`d1_test_hooks_enabled` defaults to `false` and is guarded to test-named compute +stacks. Keep it disabled for normal live-ready service state. ## Validation diff --git a/terraform/main.tf b/terraform/main.tf index 02bee91..b3e5651 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -45,7 +45,6 @@ module "compute" { d1_runtime_desired_count = var.d1_runtime_desired_count do_runtime_desired_count = var.do_runtime_desired_count d1_test_hooks_enabled = var.d1_test_hooks_enabled - do_test_hooks_enabled = var.do_test_hooks_enabled scheduler_desired_count = var.scheduler_desired_count workflows_desired_count = var.workflows_desired_count gateway_cpu = var.gateway_cpu diff --git a/terraform/modules/compute/do_runtime_service.tf b/terraform/modules/compute/do_runtime_service.tf index 47664c2..ef5fee2 100644 --- a/terraform/modules/compute/do_runtime_service.tf +++ b/terraform/modules/compute/do_runtime_service.tf @@ -79,9 +79,7 @@ resource "aws_ecs_task_definition" "do_runtime" { { name = "DO_TASK_CONTAINER_NAME", value = "do-runtime" }, { name = "DO_OWNER_TTL_SECONDS", value = "120" }, { name = "D1_QUERY_TIMEOUT_MS", value = "30000" }, - ], var.do_test_hooks_enabled ? [ - { name = "DO_TEST_HOOKS", value = "1" }, - ] : [], local.r2_s3_env) + ], local.r2_s3_env) secrets = concat([ { name = "R2_S3_ACCESS_KEY_ID", valueFrom = "${var.runtime_r2_secret_arn}:access_key_id::" }, @@ -116,10 +114,6 @@ resource "aws_ecs_task_definition" "do_runtime" { error_message = "do_runtime_container_memory must leave task-level memory reservation for the redis-proxy sidecar and additional task headroom." } - precondition { - condition = !var.do_test_hooks_enabled || can(regex("(^|-)test($|-)", var.name)) - error_message = "do_test_hooks_enabled may only be enabled for test-named compute stacks." - } } } diff --git a/terraform/modules/compute/scheduler_service.tf b/terraform/modules/compute/scheduler_service.tf index 6656660..4d9ec98 100644 --- a/terraform/modules/compute/scheduler_service.tf +++ b/terraform/modules/compute/scheduler_service.tf @@ -17,7 +17,7 @@ resource "aws_ecs_task_definition" "scheduler" { image = var.rust_image essential = true command = ["/scheduler"] - stopTimeout = 20 + stopTimeout = 30 environment = [ { name = "REDIS_URL", value = "redis://${local.redis_addr}" }, diff --git a/terraform/modules/compute/security_groups.tf b/terraform/modules/compute/security_groups.tf index 5ccef91..f901ecb 100644 --- a/terraform/modules/compute/security_groups.tf +++ b/terraform/modules/compute/security_groups.tf @@ -36,6 +36,10 @@ resource "aws_security_group" "runtime" { } } +# ECS intentionally shares this SG across user, system, D1, and DO tasks. Rules +# sourced from or targeting it are therefore coarser than Kubernetes per-component +# NetworkPolicies; splitting those caller sets requires splitting this SG first. + # Range covers :8081 (loader sockets) + :8082 (system-runtime control). resource "aws_security_group_rule" "runtime_from_gateway" { type = "ingress" diff --git a/terraform/modules/compute/variables.tf b/terraform/modules/compute/variables.tf index 86e5046..7ff69f4 100644 --- a/terraform/modules/compute/variables.tf +++ b/terraform/modules/compute/variables.tf @@ -44,7 +44,6 @@ variable "runtime_desired_count" { type = number } variable "d1_runtime_desired_count" { type = number } variable "do_runtime_desired_count" { type = number } variable "d1_test_hooks_enabled" { type = bool } -variable "do_test_hooks_enabled" { type = bool } variable "scheduler_desired_count" { type = number } variable "workflows_desired_count" { type = number } variable "gateway_cpu" { type = number } diff --git a/terraform/modules/compute/workflows_service.tf b/terraform/modules/compute/workflows_service.tf index 0ad489c..c1192e9 100644 --- a/terraform/modules/compute/workflows_service.tf +++ b/terraform/modules/compute/workflows_service.tf @@ -17,7 +17,7 @@ resource "aws_ecs_task_definition" "workflows" { image = var.rust_image essential = true command = ["/workflows"] - stopTimeout = 20 + stopTimeout = 30 portMappings = [{ name = "workflows-http" diff --git a/terraform/terraform.tfvars.example b/terraform/terraform.tfvars.example index 38fb77f..53be769 100644 --- a/terraform/terraform.tfvars.example +++ b/terraform/terraform.tfvars.example @@ -21,7 +21,6 @@ alb_https_listener_arn = "arn:aws:elasticloadbalancing:ap-east-1:123456789012:li alb_security_group_id = "sg-xxxxxxxxxxxxxxxxx" d1_test_hooks_enabled = false -do_test_hooks_enabled = false # Stateless gateway/user-runtime/system-runtime keep at least one on-demand # Fargate task and place overflow according to these weights. Stateful runtimes, diff --git a/terraform/variables.tf b/terraform/variables.tf index 6213a71..c619a2e 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -165,12 +165,6 @@ variable "d1_test_hooks_enabled" { description = "Enable D1 internal test hooks in d1-runtime. Keep false outside disposable/test environments." } -variable "do_test_hooks_enabled" { - type = bool - default = false - description = "Enable DO internal test hooks in do-runtime. Keep false outside disposable/test environments." -} - variable "gateway_cpu" { type = number default = 512 diff --git a/test-workers/do-rpc/src/index.js b/test-workers/do-rpc/src/index.js index 8d24117..5019f5e 100644 --- a/test-workers/do-rpc/src/index.js +++ b/test-workers/do-rpc/src/index.js @@ -1,4 +1,6 @@ import { DurableObject } from "cloudflare:workers"; +import * as hostWrapperRuntime from "./_wdl-host-wrapper-runtime.js"; +import wrappedWorker from "./_wdl-wrapper.js"; export class Room extends DurableObject { constructor(ctx, env) { @@ -44,11 +46,45 @@ export class Room extends DurableObject { result: await target.addMessage(text, { role: "peer" }), }; } + + async forwardRequestId(targetName) { + this.env.ROOM.requestId = () => "tenant-rid"; + const target = this.env.ROOM.get(this.env.ROOM.idFromName(targetName)); + const tenantContext = hostWrapperRuntime.createRequestContext(); + return tenantContext.runWithRequestContext("tenant-rid", async () => { + const response = await target.fetch(new Request("https://do.internal/request-id")); + return response.text(); + }); + } + + async nestedForwardRequestId(targetName) { + const response = await this.fetch(new Request( + `https://do.internal/forward-request-id?to=${encodeURIComponent(targetName)}` + )); + return response.text(); + } + + fetch(request) { + const url = new URL(request.url); + if (url.pathname === "/forward-request-id") { + const target = this.env.ROOM.get(this.env.ROOM.idFromName( + url.searchParams.get("to") || "peer" + )); + return target.fetch(new Request("https://do.internal/request-id")); + } + return new Response(request.headers.get("x-request-id") || ""); + } } export default { async fetch(request, env) { const url = new URL(request.url); + if (url.pathname === "/reenter-request-id" && url.searchParams.get("nested") !== "1") { + url.searchParams.set("nested", "1"); + return wrappedWorker.fetch(new Request(url, { + headers: { "x-request-id": "tenant-rid" }, + }), env, {}); + } const id = env.ROOM.idFromName(url.searchParams.get("name") || "main"); const stub = env.ROOM.get(id); if (url.pathname === "/fail") { @@ -85,6 +121,21 @@ export default { "forwarded" )); } + if (url.pathname === "/request-id") { + return Response.json({ + requestId: await stub.forwardRequestId(url.searchParams.get("to") || "peer"), + }); + } + if (url.pathname === "/reenter-request-id") { + return Response.json({ + requestId: await stub.forwardRequestId(url.searchParams.get("to") || "peer"), + }); + } + if (url.pathname === "/nested-request-id") { + return Response.json({ + requestId: await stub.nestedForwardRequestId(url.searchParams.get("to") || "peer"), + }); + } return Response.json(await stub.addMessage("hello", { role: "user" })); }, }; diff --git a/tests/fixtures/cross-language-identity.json b/tests/fixtures/cross-language-identity.json index 4c6d227..248df6f 100644 --- a/tests/fixtures/cross-language-identity.json +++ b/tests/fixtures/cross-language-identity.json @@ -50,6 +50,20 @@ { "value": "jobs:tail", "valid": false }, { "value": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "valid": false } ], + "kvIds": [ + { "value": "cache", "valid": true }, + { "value": "9cache", "valid": true }, + { "value": "session-store", "valid": true }, + { "value": "a-", "valid": true }, + { "value": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "valid": true }, + { "value": "", "valid": false }, + { "value": "-cache", "valid": false }, + { "value": "Cache", "valid": false }, + { "value": "cache_id", "valid": false }, + { "value": "cache:id", "valid": false }, + { "value": "cache/id", "valid": false }, + { "value": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "valid": false } + ], "workflowInstanceIds": [ { "value": "inst-1", "valid": true }, { "value": "a_1", "valid": true }, diff --git a/tests/fixtures/do-alarm-identity.json b/tests/fixtures/do-alarm-identity.json new file mode 100644 index 0000000..c846749 --- /dev/null +++ b/tests/fixtures/do-alarm-identity.json @@ -0,0 +1,142 @@ +{ + "maxBytes": 512, + "hostShardCount": 16, + "cases": [ + { + "name": "canonical", + "doStorageId": "do_abc-123", + "className": "AlarmCounter", + "objectName": "alpha", + "token": "row-token", + "valid": true + }, + { + "name": "empty-object-name", + "doStorageId": "do_abc", + "className": "AlarmCounter", + "objectName": "", + "token": "row-token", + "valid": false + }, + { + "name": "empty-token", + "doStorageId": "do_abc", + "className": "AlarmCounter", + "objectName": "alpha", + "token": "", + "valid": false + }, + { + "name": "storage-id-grammar", + "doStorageId": "bad/storage", + "className": "AlarmCounter", + "objectName": "alpha", + "token": "row-token", + "valid": false + }, + { + "name": "class-name-grammar", + "doStorageId": "do_abc", + "className": "1Alarm", + "objectName": "alpha", + "token": "row-token", + "valid": false + }, + { + "name": "one-digit-shard-aggregate-max", + "doStorageId": { "repeat": "a", "count": 496 }, + "className": "ChatRoom", + "objectName": "zero", + "token": "row-token", + "valid": true + }, + { + "name": "one-digit-shard-aggregate-over-limit", + "doStorageId": { "repeat": "a", "count": 497 }, + "className": "ChatRoom", + "objectName": "zero", + "token": "row-token", + "valid": false + }, + { + "name": "two-digit-shard-aggregate-max", + "doStorageId": { "repeat": "a", "count": 495 }, + "className": "ChatRoom", + "objectName": "alpha", + "token": "row-token", + "valid": true + }, + { + "name": "two-digit-shard-aggregate-over-limit", + "doStorageId": { "repeat": "a", "count": 496 }, + "className": "ChatRoom", + "objectName": "alpha", + "token": "row-token", + "valid": false + }, + { + "name": "object-name-control-character", + "doStorageId": "do_abc", + "className": "AlarmCounter", + "objectName": "bad\u000bname", + "token": "row-token", + "valid": false + }, + { + "name": "object-name-supplementary-scalar", + "doStorageId": "do_abc", + "className": "AlarmCounter", + "objectName": "\ud83d\ude00", + "token": "row-token", + "valid": true + }, + { + "name": "supplementary-scalar-two-digit-shard-over-limit", + "doStorageId": { "repeat": "a", "count": 496 }, + "className": "ChatRoom", + "objectName": "\ud83d\ude80", + "token": "row-token", + "valid": false + }, + { + "name": "object-name-utf8-max", + "doStorageId": "do_abc", + "className": "AlarmCounter", + "objectName": { "repeat": "\u00e9", "count": 256 }, + "token": "row-token", + "valid": true + }, + { + "name": "object-name-utf8-over-limit", + "doStorageId": "do_abc", + "className": "AlarmCounter", + "objectName": { "repeat": "\u00e9", "count": 257 }, + "token": "row-token", + "valid": false + }, + { + "name": "token-control-character", + "doStorageId": "do_abc", + "className": "AlarmCounter", + "objectName": "alpha", + "token": "bad\u000btoken", + "valid": false + }, + { + "name": "token-utf8-max", + "doStorageId": "do_abc", + "className": "AlarmCounter", + "objectName": "alpha", + "token": { "repeat": "\u00e9", "count": 256 }, + "valid": true + }, + { + "name": "token-utf8-over-limit", + "doStorageId": "do_abc", + "className": "AlarmCounter", + "objectName": "alpha", + "token": { "repeat": "\u00e9", "count": 257 }, + "valid": false + } + ] +} diff --git a/tests/fixtures/do-intrinsic-guard-worker.mjs.txt b/tests/fixtures/do-intrinsic-guard-worker.mjs.txt new file mode 100644 index 0000000..a981a52 --- /dev/null +++ b/tests/fixtures/do-intrinsic-guard-worker.mjs.txt @@ -0,0 +1,70 @@ +const iteratorSymbol = Symbol.iterator; +const originalArrayIterator = Array.prototype[iteratorSymbol]; +const originalFunctionToString = Function.prototype.toString; +const originalObjectDefineProperty = Object.defineProperty; +const originalObjectEntries = Object.entries; +const originalObjectKeys = Object.keys; +const originalPromiseThen = Promise.prototype.then; +const originalProxy = Proxy; +const originalReflectGet = Reflect.get; +const originalSymbol = Symbol; + +Array.prototype[iteratorSymbol] = function* () {}; +Function.prototype.toString = () => "function fetch() {}"; +Object.defineProperty = () => {}; +Object.entries = () => []; +Object.keys = () => []; +Promise.prototype.then = () => { throw new Error("tenant Promise.then must not run"); }; +globalThis.Proxy = class BrokenProxy { constructor(target) { return target; } }; +Reflect.get = () => undefined; +globalThis.Symbol = () => "BYPASS"; + +function restoreIntrinsics() { + globalThis.Symbol = originalSymbol; + Reflect.get = originalReflectGet; + globalThis.Proxy = originalProxy; + Object.keys = originalObjectKeys; + Object.entries = originalObjectEntries; + Object.defineProperty = originalObjectDefineProperty; + Promise.prototype.then = originalPromiseThen; + Function.prototype.toString = originalFunctionToString; +} + +function restorePromiseThen() { + Promise.prototype.then = originalPromiseThen; +} + +function restoreArrayIterator() { + Array.prototype[iteratorSymbol] = originalArrayIterator; +} + +export class Room { + async fetch(request) { + queueMicrotask(restorePromiseThen); + return new Response(await request.text()); + } +} + +export default { + async fetch(_request, env) { + restoreIntrinsics(); + const room = env.ROOM.get(env.ROOM.idFromName("guard")); + const roomResponse = await room.fetch("https://room.local/", { + method: "POST", + body: "room-ok", + }); + const ownerMetadataVisible = + roomResponse.headers.has("x-wdl-do-owner-task-id") || + roomResponse.headers.has("x-wdl-do-owner-endpoint") || + roomResponse.headers.has("x-wdl-do-ownership-error"); + restoreArrayIterator(); + return Response.json({ + room: await roomResponse.text(), + roomFacade: env.ROOM.constructor.name, + ownerMetadataVisible, + doBackendVisible: "__WDL_DO_BACKEND__" in env, + ownerNetworkVisible: "__WDL_DO_OWNER_NETWORK__" in env, + workflowsBackendVisible: "__WDL_WORKFLOWS_BACKEND__" in env, + }); + }, +}; diff --git a/tests/fixtures/internal-auth-contract.json b/tests/fixtures/internal-auth-contract.json new file mode 100644 index 0000000..3f4a065 --- /dev/null +++ b/tests/fixtures/internal-auth-contract.json @@ -0,0 +1,39 @@ +{ + "header": "x-wdl-internal-auth", + "currentEnv": "WDL_INTERNAL_AUTH_TOKEN", + "previousEnv": "WDL_INTERNAL_AUTH_PREVIOUS_TOKEN", + "failure": { + "status": 401, + "error": "internal_auth_failed", + "message": "Internal authentication failed" + }, + "tokenCases": [ + { "value": "current", "accepted": true }, + { "value": "token-._~+/=", "accepted": true }, + { "value": "", "accepted": false }, + { "value": " leading", "accepted": false }, + { "value": "trailing ", "accepted": false }, + { "value": "inner space", "accepted": false }, + { "value": "token\tvalue", "accepted": false }, + { "value": "token\nvalue", "accepted": false }, + { "value": "token\u0000value", "accepted": false }, + { "value": "token\u007fvalue", "accepted": false }, + { "value": "token,other", "accepted": false }, + { "value": "tokén", "accepted": false } + ], + "rotationCases": [ + { "actual": "current", "current": "current", "previous": null, "accepted": true }, + { "actual": "current", "current": "current", "previous": "previous", "accepted": true }, + { "actual": "previous", "current": "current", "previous": "previous", "accepted": true }, + { "actual": "wrong", "current": "current", "previous": "previous", "accepted": false }, + { "actual": null, "current": "current", "previous": "previous", "accepted": false }, + { "actual": "previous", "current": "current", "previous": null, "accepted": false } + ], + "headerCases": [ + { "values": ["current"], "current": "current", "previous": "previous", "accepted": true }, + { "values": ["previous"], "current": "current", "previous": "previous", "accepted": true }, + { "values": ["current", "wrong"], "current": "current", "previous": "previous", "accepted": false }, + { "values": ["wrong", "current"], "current": "current", "previous": "previous", "accepted": false }, + { "values": ["current", "current"], "current": "current", "previous": null, "accepted": false } + ] +} diff --git a/tests/fixtures/observability-contract.json b/tests/fixtures/observability-contract.json index b3698ad..fef5f54 100644 --- a/tests/fixtures/observability-contract.json +++ b/tests/fixtures/observability-contract.json @@ -18,5 +18,15 @@ { "raw": "a\rb", "escaped": "a\\rb" }, { "raw": "a\"b", "escaped": "a\\\"b" }, { "raw": "\r\n\"\\", "escaped": "\\r\\n\\\"\\\\" } - ] + ], + "logEnvelope": { + "orderedKeys": ["ts", "service", "level", "event"], + "timestampShape": "0000-00-00T00:00:00.000Z", + "levels": [ + { "name": "debug", "priority": 10, "stream": "stdout" }, + { "name": "info", "priority": 20, "stream": "stdout" }, + { "name": "warn", "priority": 30, "stream": "stdout" }, + { "name": "error", "priority": 40, "stream": "stderr" } + ] + } } diff --git a/tests/fixtures/queue-key-parse.json b/tests/fixtures/queue-key-parse.json new file mode 100644 index 0000000..175bb3f --- /dev/null +++ b/tests/fixtures/queue-key-parse.json @@ -0,0 +1,35 @@ +{ + "stream": [ + { "key": "queue:demo:jobs:s", "parsed": { "ns": "demo", "queue": "jobs" } }, + { "key": "queue:__platform__:jobs:s", "parsed": { "ns": "__platform__", "queue": "jobs" } }, + { "key": "queue:t:my-queue-1:s", "parsed": { "ns": "t", "queue": "my-queue-1" } }, + { "key": "queue:demo:jobs", "parsed": null }, + { "key": "queue-delayed:demo:jobs", "parsed": null }, + { "key": "queue::jobs:s", "parsed": null }, + { "key": "queue:admin:jobs:s", "parsed": null }, + { "key": "queue:__community__:jobs:s", "parsed": null }, + { "key": "queue:demo:Jobs:s", "parsed": null }, + { "key": "queue:demo:jobs:extra:s", "parsed": null }, + { "key": "notaqueue:demo:jobs:s", "parsed": null }, + { "key": "", "parsed": null } + ], + "delayed": [ + { "key": "queue-delayed:demo:jobs", "parsed": { "ns": "demo", "queue": "jobs" } }, + { "key": "queue-delayed:__system__:jobs-1", "parsed": { "ns": "__system__", "queue": "jobs-1" } }, + { "key": "queue:demo:jobs:s", "parsed": null }, + { "key": "queue-delayed::jobs", "parsed": null }, + { "key": "queue-delayed:demo:", "parsed": null }, + { "key": "queue-delayed:demo:Jobs", "parsed": null }, + { "key": "queue-delayed:demo:jobs:extra", "parsed": null } + ], + "consumer": [ + { "key": "queue-consumer:demo:jobs", "parsed": { "ns": "demo", "queue": "jobs" } }, + { "key": "queue-consumer:__system__:jobs-1", "parsed": { "ns": "__system__", "queue": "jobs-1" } }, + { "key": "queue-consumer:__platform__:jobs-1", "parsed": null }, + { "key": "queue-consumer:demo", "parsed": null }, + { "key": "queue-consumer:", "parsed": null }, + { "key": "queue-consumer:admin:jobs", "parsed": null }, + { "key": "queue-consumer:demo:jobs:extra", "parsed": null }, + { "key": "other:demo:jobs", "parsed": null } + ] +} diff --git a/tests/fixtures/request-id-sanitizer.json b/tests/fixtures/request-id-sanitizer.json index 3164db7..1d4e77a 100644 --- a/tests/fixtures/request-id-sanitizer.json +++ b/tests/fixtures/request-id-sanitizer.json @@ -7,7 +7,15 @@ { "raw": "a\"b", "sanitized": null }, { "raw": "a\\b", "sanitized": null }, { "raw": "ok\nbad", "sanitized": null }, + { "raw": "\u000bxyz", "sanitized": null }, { "raw": "ok\u0085bad", "sanitized": null }, + { "raw": "\u0085rid", "sanitized": null }, + { "raw": "rid\u0085", "sanitized": null }, + { "raw": "ok\u00a0bad", "sanitized": null }, + { "raw": "ok\u2028bad", "sanitized": null }, + { "raw": "ok\u3000bad", "sanitized": null }, + { "raw": "ok\ufeffbad", "sanitized": null }, + { "raw": "caf\u00e9", "sanitized": null }, { "raw": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "sanitized": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, { "raw": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "sanitized": null } ] diff --git a/tests/fixtures/scheduler-projection-contract.json b/tests/fixtures/scheduler-projection-contract.json new file mode 100644 index 0000000..ff84136 --- /dev/null +++ b/tests/fixtures/scheduler-projection-contract.json @@ -0,0 +1,48 @@ +{ + "cron": { + "workerIndexKey": "cron:index:workers", + "ns": "tenant", + "worker": "worker", + "workerKey": "crons:tenant:worker", + "slotMs": 1778856600000, + "slotKey": "cron-slot:1778856600000", + "slotExpireAt": 1778857200, + "cronId": "cron-1", + "gen": 7, + "reference": "tenant:worker:cron-1:7", + "meta": { + "version": "v3", + "seq": 7, + "json": "{\"version\":\"v3\",\"seq\":7}" + }, + "entry": { + "cron": "*/5 * * * *", + "timezone": "UTC", + "gen": 7, + "json": "{\"cron\":\"*/5 * * * *\",\"timezone\":\"UTC\",\"gen\":7}" + } + }, + "queueConsumer": { + "ns": "tenant", + "queue": "jobs", + "worker": "worker", + "version": "v3", + "input": { + "queue": "jobs", + "maxBatchSize": 12, + "maxBatchTimeoutMs": 250, + "maxRetries": 4, + "deadLetterQueue": "jobs-dlq", + "retryDelaySeconds": 17 + }, + "fields": { + "worker": "worker", + "version": "v3", + "max_batch_size": "12", + "max_batch_timeout_ms": "250", + "max_retries": "4", + "dead_letter_queue": "jobs-dlq", + "retry_delay_secs": "17" + } + } +} diff --git a/tests/fixtures/secret-envelope-parity.json b/tests/fixtures/secret-envelope-parity.json new file mode 100644 index 0000000..be1a176 --- /dev/null +++ b/tests/fixtures/secret-envelope-parity.json @@ -0,0 +1,74 @@ +{ + "provider": { + "kid": "local:test:secret-envelope:v1", + "localKeyB64": "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=" + }, + "vectors": [ + { + "name": "value", + "plaintext": "sensitive-value", + "hashKey": "secrets:demo:api", + "fieldName": "TOKEN", + "randomStart": 0, + "envelope": "WDL-ENC:{\"v\":1,\"alg\":\"AES-256-GCM\",\"kid\":\"local:test:secret-envelope:v1\",\"edek\":\"ICEiIyQlJicoKSorrmiFFNwSNL789zDTZysVGjsTpksuXqulg0Mt7JrBiLTG0wE1moqP31Jd+edymdiU\",\"iv\":\"LC0uLzAxMjM0NTY3\",\"ct\":\"IZhLZQvljbpT5euHV5YP\",\"tag\":\"0wuyjfzeZfuY5hD3HRoKdg==\"}" + }, + { + "name": "empty", + "plaintext": "", + "hashKey": "secrets:demo:api", + "fieldName": "EMPTY", + "randomStart": 0, + "envelope": "WDL-ENC:{\"v\":1,\"alg\":\"AES-256-GCM\",\"kid\":\"local:test:secret-envelope:v1\",\"edek\":\"ICEiIyQlJicoKSorrmiFFNwSNL789zDTZysVGjsTpksuXqulg0Mt7JrBiLSmHuPVe4uS7JepH053l0Yt\",\"iv\":\"LC0uLzAxMjM0NTY3\",\"ct\":\"\",\"tag\":\"TTVUTxM6DfSV+VZ/AODK1w==\"}" + } + ], + "rejections": [ + { + "name": "wrong-hash", + "envelope": "WDL-ENC:{\"v\":1,\"alg\":\"AES-256-GCM\",\"kid\":\"local:test:secret-envelope:v1\",\"edek\":\"ICEiIyQlJicoKSorrmiFFNwSNL789zDTZysVGjsTpksuXqulg0Mt7JrBiLTG0wE1moqP31Jd+edymdiU\",\"iv\":\"LC0uLzAxMjM0NTY3\",\"ct\":\"IZhLZQvljbpT5euHV5YP\",\"tag\":\"0wuyjfzeZfuY5hD3HRoKdg==\"}", + "hashKey": "secrets:other:api", + "fieldName": "TOKEN", + "configuredKid": "local:test:secret-envelope:v1", + "jsErrorCode": "secret_decrypt_failed" + }, + { + "name": "wrong-field", + "envelope": "WDL-ENC:{\"v\":1,\"alg\":\"AES-256-GCM\",\"kid\":\"local:test:secret-envelope:v1\",\"edek\":\"ICEiIyQlJicoKSorrmiFFNwSNL789zDTZysVGjsTpksuXqulg0Mt7JrBiLTG0wE1moqP31Jd+edymdiU\",\"iv\":\"LC0uLzAxMjM0NTY3\",\"ct\":\"IZhLZQvljbpT5euHV5YP\",\"tag\":\"0wuyjfzeZfuY5hD3HRoKdg==\"}", + "hashKey": "secrets:demo:api", + "fieldName": "OTHER", + "configuredKid": "local:test:secret-envelope:v1", + "jsErrorCode": "secret_decrypt_failed" + }, + { + "name": "wrong-kid", + "envelope": "WDL-ENC:{\"v\":1,\"alg\":\"AES-256-GCM\",\"kid\":\"local:test:secret-envelope:v1\",\"edek\":\"ICEiIyQlJicoKSorrmiFFNwSNL789zDTZysVGjsTpksuXqulg0Mt7JrBiLTG0wE1moqP31Jd+edymdiU\",\"iv\":\"LC0uLzAxMjM0NTY3\",\"ct\":\"IZhLZQvljbpT5euHV5YP\",\"tag\":\"0wuyjfzeZfuY5hD3HRoKdg==\"}", + "hashKey": "secrets:demo:api", + "fieldName": "TOKEN", + "configuredKid": "local:test:secret-envelope:v2", + "jsErrorCode": "unknown_kid" + }, + { + "name": "non-canonical-json", + "envelope": "WDL-ENC:{ \"v\":1,\"alg\":\"AES-256-GCM\",\"kid\":\"local:test:secret-envelope:v1\",\"edek\":\"ICEiIyQlJicoKSorrmiFFNwSNL789zDTZysVGjsTpksuXqulg0Mt7JrBiLTG0wE1moqP31Jd+edymdiU\",\"iv\":\"LC0uLzAxMjM0NTY3\",\"ct\":\"IZhLZQvljbpT5euHV5YP\",\"tag\":\"0wuyjfzeZfuY5hD3HRoKdg==\"}", + "hashKey": "secrets:demo:api", + "fieldName": "TOKEN", + "configuredKid": "local:test:secret-envelope:v1", + "jsErrorCode": "invalid_envelope" + }, + { + "name": "non-canonical-field-order", + "envelope": "WDL-ENC:{\"alg\":\"AES-256-GCM\",\"v\":1,\"kid\":\"local:test:secret-envelope:v1\",\"edek\":\"ICEiIyQlJicoKSorrmiFFNwSNL789zDTZysVGjsTpksuXqulg0Mt7JrBiLTG0wE1moqP31Jd+edymdiU\",\"iv\":\"LC0uLzAxMjM0NTY3\",\"ct\":\"IZhLZQvljbpT5euHV5YP\",\"tag\":\"0wuyjfzeZfuY5hD3HRoKdg==\"}", + "hashKey": "secrets:demo:api", + "fieldName": "TOKEN", + "configuredKid": "local:test:secret-envelope:v1", + "jsErrorCode": "invalid_envelope" + }, + { + "name": "non-canonical-base64", + "envelope": "WDL-ENC:{\"v\":1,\"alg\":\"AES-256-GCM\",\"kid\":\"local:test:secret-envelope:v1\",\"edek\":\"ICEiIyQlJicoKSorrmiFFNwSNL789zDTZysVGjsTpksuXqulg0Mt7JrBiLTG0wE1moqP31Jd+edymdiU\",\"iv\":\"LC0uLzAxMjM0NTY3\",\"ct\":\"IZhLZQvljbpT5euHV5YP\",\"tag\":\"0wuyjfzeZfuY5hD3HRoKdg\"}", + "hashKey": "secrets:demo:api", + "fieldName": "TOKEN", + "configuredKid": "local:test:secret-envelope:v1", + "jsErrorCode": "invalid_envelope" + } + ] +} diff --git a/tests/fixtures/version-tags.json b/tests/fixtures/version-tags.json new file mode 100644 index 0000000..e673116 --- /dev/null +++ b/tests/fixtures/version-tags.json @@ -0,0 +1,16 @@ +{ + "cases": [ + { "tag": "v1", "parsed": 1 }, + { "tag": "v42", "parsed": 42 }, + { "tag": "v9007199254740991", "parsed": 9007199254740991 }, + { "tag": "", "parsed": null }, + { "tag": "v", "parsed": null }, + { "tag": "v0", "parsed": null }, + { "tag": "v01", "parsed": null }, + { "tag": "V1", "parsed": null }, + { "tag": "v1a", "parsed": null }, + { "tag": "v9007199254740992", "parsed": null }, + { "tag": "v18446744073709551615", "parsed": null }, + { "tag": "v999999999999999999999999999999", "parsed": null } + ] +} diff --git a/tests/fixtures/workflow-limits.json b/tests/fixtures/workflow-limits.json new file mode 100644 index 0000000..804b6bf --- /dev/null +++ b/tests/fixtures/workflow-limits.json @@ -0,0 +1,6 @@ +{ + "resultBytesMax": 1048576, + "backendRequestBytesMax": 2097152, + "createBatchMax": 100, + "payloadTooLargeCode": "workflow_payload_too_large" +} diff --git a/tests/helpers/control-handler-harness.js b/tests/helpers/control-handler-harness.js index 8145e92..b290f02 100644 --- a/tests/helpers/control-handler-harness.js +++ b/tests/helpers/control-handler-harness.js @@ -4,7 +4,6 @@ import { importSpecifierReplacements, } from "./load-shared-module.js"; import { createFakeRedis } from "./mocks/fake-redis.js"; -import { OBSERVABILITY_NOOP_URL } from "./mocks/observability.js"; const DEFAULT_GLOBAL_NAME = "__controlHandlerHarnessState"; @@ -79,7 +78,6 @@ export const metrics = { increment(...args) { harnessState().metrics.increment?.(...args); }, observe(...args) { harnessState().metrics.observe?.(...args); }, }; -export { formatError } from ${JSON.stringify(OBSERVABILITY_NOOP_URL)}; ${extraSource} `); } diff --git a/tests/helpers/control-shared-stub.js b/tests/helpers/control-shared-stub.js index dccc3f2..2563f66 100644 --- a/tests/helpers/control-shared-stub.js +++ b/tests/helpers/control-shared-stub.js @@ -2,100 +2,65 @@ // tail so each test wires its own redis fakes without re-declaring the base // helpers. -import { moduleDataUrl } from "./load-shared-module.js"; +import { + freshRepositoryModuleDataUrl, + moduleDataUrl, + repositoryFileUrl, +} from "./load-shared-module.js"; +import { sharedRedisStubUrl } from "./mocks/fake-redis.js"; +import { OBSERVABILITY_NOOP_URL } from "./mocks/observability.js"; + +const SHARED_BOUNDED_BODY_URL = repositoryFileUrl("shared/bounded-body.js"); +const SHARED_ERRORS_URL = repositoryFileUrl("shared/errors.js"); +const SHARED_INTERNAL_AUTH_URL = repositoryFileUrl("shared/internal-auth.js"); +const SHARED_RANDOM_ID_URL = repositoryFileUrl("shared/random-id.js"); +const SHARED_RESPOND_URL = repositoryFileUrl("shared/respond.js"); +const SHARED_OPTIMISTIC_RETRY_URL = repositoryFileUrl("shared/optimistic-retry.js"); +const CONTROL_ERRORS_URL = freshRepositoryModuleDataUrl("control/errors.js", [ + [/from "shared-errors"/g, `from ${JSON.stringify(SHARED_ERRORS_URL)}`], + [/from "shared-respond"/g, `from ${JSON.stringify(SHARED_RESPOND_URL)}`], +]); +const CONTROL_WORKFLOWS_CLIENT_URL = freshRepositoryModuleDataUrl("control/workflows-client.js", [ + [/from "shared-errors"/g, `from ${JSON.stringify(SHARED_ERRORS_URL)}`], + [/from "control-errors"/g, `from ${JSON.stringify(CONTROL_ERRORS_URL)}`], +]); +const CONTROL_OPTIMISTIC_REDIS_URL = sharedRedisStubUrl(); +const CONTROL_OPTIMISTIC_URL = freshRepositoryModuleDataUrl("control/optimistic.js", [ + [/from "shared-redis"/g, `from ${JSON.stringify(CONTROL_OPTIMISTIC_REDIS_URL)}`], + [/from "shared-optimistic-retry"/g, `from ${JSON.stringify(SHARED_OPTIMISTIC_RETRY_URL)}`], +]); +const CONTROL_JSON_BODY_URL = freshRepositoryModuleDataUrl("control/json-body.js", [ + [/from "shared-bounded-body"/g, `from ${JSON.stringify(SHARED_BOUNDED_BODY_URL)}`], + [/from "shared-respond"/g, `from ${JSON.stringify(SHARED_RESPOND_URL)}`], +]); const CONTROL_SHARED_BASE = ` -export function jsonResponse(status, data, extraHeaders = {}) { - return new Response(JSON.stringify(data), { status, headers: { "content-type": "application/json", ...extraHeaders } }); -} -function sanitizeJsonErrorDetails(value) { - if (Array.isArray(value)) return value; - if (!value || typeof value !== "object") return value; - const out = {}; - for (const [key, entry] of Object.entries(value)) { - if (key === "error" || key === "message" || key === "reason") continue; - if (entry !== undefined) { - Object.defineProperty(out, key, { - value: entry, - enumerable: true, - configurable: true, - writable: true, - }); - } - } - return Object.keys(out).length === 0 ? undefined : out; -} -export function jsonError(status, error, message, details = {}, extraHeaders = {}) { - const body = { error }; - if (message) body.message = message; - const sanitized = sanitizeJsonErrorDetails(details); - const safeDetails = sanitized && typeof sanitized === "object" && !Array.isArray(sanitized) ? sanitized : {}; - return jsonResponse(status, { ...safeDetails, ...body }, extraHeaders); -} -export async function readJsonBody(request, { requireObject = false, allowEmpty = false, maxBytes = 1024 * 1024 } = {}) { - let body; - try { - const declared = Number(request.headers.get("content-length") || 0); - if (Number.isFinite(declared) && declared > maxBytes) { - return { response: jsonError(413, "request_body_too_large", "Body must be at most " + maxBytes + " bytes") }; - } - const text = await request.text(); - if (Buffer.byteLength(text, "utf8") > maxBytes) { - return { response: jsonError(413, "request_body_too_large", "Body must be at most " + maxBytes + " bytes") }; - } - if (text === "") { - if (!allowEmpty) { - return { response: jsonError(400, "invalid_json", "Body must be valid JSON") }; - } - body = {}; - } else { - body = JSON.parse(text); - } - } catch { - return { response: jsonError(400, "invalid_json", "Body must be valid JSON") }; - } - if (requireObject && (!body || typeof body !== "object" || Array.isArray(body))) { - return { response: jsonError(400, "invalid_json_object", "Body must be a JSON object") }; - } - return { body }; -} -export class ControlAbort extends Error { - constructor(status, code, details = {}) { - super(details?.message || code); - this.status = status; - this.code = code; - this.details = details; - } -} -export function codedErrorResponse(err, fallbackCode = "internal_error", extraDetails = {}) { - const record = err && typeof err === "object" && !Array.isArray(err) ? err : {}; - const details = record.details && typeof record.details === "object" && !Array.isArray(record.details) - ? record.details - : {}; - const status = typeof record.status === "number" ? record.status : 500; - const code = typeof record.code === "string" && record.code ? record.code : fallbackCode; - const recordMessage = typeof record.message === "string" && record.message ? record.message : undefined; - const message = err instanceof ControlAbort - ? (typeof err.details.message === "string" && err.details.message) || err.message || code - : recordMessage || (typeof details.message === "string" && details.message) || code; - return jsonError( - status, - code, - message, - { ...details, ...extraDetails }, - ); -} -export function controlAbortResponse(err, extraDetails = {}) { - return codedErrorResponse(err, err.code, extraDetails); -} -export function errMessage(err) { - return err instanceof Error ? err.message : String(err); -} -export function randomHex(bytes = 16) { - const data = new Uint8Array(bytes); - crypto.getRandomValues(data); - return Array.from(data, (b) => b.toString(16).padStart(2, "0")).join(""); -} +import { jsonError, jsonResponse, sanitizeJsonErrorDetails } from ${JSON.stringify(SHARED_RESPOND_URL)}; +import { createPostWorkflowsInternal } from ${JSON.stringify(CONTROL_WORKFLOWS_CLIENT_URL)}; +import { ControlAbort, controlAbortLogDetails, codedErrorLogFields, codedErrorResponse, controlAbortResponse, secretEnvelopeErrorResponse } from ${JSON.stringify(CONTROL_ERRORS_URL)}; +import { runOptimistic, withOptimisticRetries } from ${JSON.stringify(CONTROL_OPTIMISTIC_URL)}; +import { readJsonBody } from ${JSON.stringify(CONTROL_JSON_BODY_URL)}; +import { errorMessage as errMessage } from ${JSON.stringify(SHARED_ERRORS_URL)}; +import { withInternalAuth } from ${JSON.stringify(SHARED_INTERNAL_AUTH_URL)}; +import { randomHex } from ${JSON.stringify(SHARED_RANDOM_ID_URL)}; +import { formatError } from ${JSON.stringify(OBSERVABILITY_NOOP_URL)}; +export { + ControlAbort, + controlAbortLogDetails, + codedErrorLogFields, + codedErrorResponse, + controlAbortResponse, + secretEnvelopeErrorResponse, + errMessage, + formatError, + jsonError, + jsonResponse, + randomHex, + readJsonBody, + runOptimistic, + sanitizeJsonErrorDetails, + withOptimisticRetries, +}; export function prefixedId(prefix, bytes = 16) { return prefix + randomHex(bytes); } @@ -125,37 +90,14 @@ export function getControlS3() { export function getControlR2() { return state.r2; } -export function getControlWorkflows() { - return state.workflows; -} export function controlInternalJsonHeaders() { - const token = state.env?.WDL_INTERNAL_AUTH_TOKEN; - return token - ? { "content-type": "application/json", "x-wdl-internal-auth": token } - : { "content-type": "application/json" }; -} -function isWatchError(err) { - return err instanceof Error && ( - err.name === "WatchError" - || err.constructor?.name === "WatchError" - ); -} -export async function runOptimistic(redis, { attempts = 5, onExhausted, onWatchError, shouldRetryResult }, fn) { - for (let attempt = 0; attempt < attempts; attempt += 1) { - try { - const result = await redis.session((session) => fn(session, attempt)); - if (shouldRetryResult?.(result, attempt)) continue; - return result; - } catch (err) { - if (isWatchError(err)) { - onWatchError?.(err, attempt); - continue; - } - throw err; - } - } - return await onExhausted(); + return withInternalAuth({ "content-type": "application/json" }, state.env); } +export const postWorkflowsInternal = createPostWorkflowsInternal({ + getWorkflows: () => state.workflows, + headers: controlInternalJsonHeaders, + getLog: () => state.log, +}); export async function recordCleanupIntentOrWarn({ cleanupIntent, cleanupTaskId, diff --git a/tests/helpers/do-owner-hint.js b/tests/helpers/do-owner-hint.js index cb66546..368daf2 100644 --- a/tests/helpers/do-owner-hint.js +++ b/tests/helpers/do-owner-hint.js @@ -31,6 +31,13 @@ export function doOwnerHintResponse(options = {}) { }); } +/** @param {string} code @param {HeadersInit | undefined} [headers] */ +export function doOwnershipErrorHeaders(code, headers = undefined) { + const out = new Headers(headers); + out.set("x-wdl-do-ownership-error", code); + return out; +} + export function tenantBodyDoOwnerHintResponse() { return Response.json({ error: "do_owner_hint", diff --git a/tests/helpers/load-auth-index.js b/tests/helpers/load-auth-index.js index 1f6b99d..0bd5220 100644 --- a/tests/helpers/load-auth-index.js +++ b/tests/helpers/load-auth-index.js @@ -12,16 +12,19 @@ import { } from "./load-shared-module.js"; import { compileSharedAuthRoles } from "./load-auth-roles.js"; import { CLOUDFLARE_WORKERS_URL } from "./mocks/cloudflare-workers.js"; +import { sharedRedisStubUrl } from "./mocks/fake-redis.js"; const SHARED_NS_URL = repositoryFileUrl("shared/ns-pattern.js"); const SHARED_AUTH_TOKEN_URL = repositoryFileUrl("shared/auth-token.js"); const SHARED_HEX_URL = repositoryFileUrl("shared/hex.js"); const SHARED_RANDOM_ID_URL = repositoryFileUrl("shared/random-id.js"); +const SHARED_REDIS_LOCK_URL = freshRepositoryModuleDataUrl("shared/redis-lock.js", [ + [/from "shared-random-id"/g, `from ${JSON.stringify(SHARED_RANDOM_ID_URL)}`], +]); const SHARED_OBSERVABILITY_URL = repositoryFileUrl("shared/observability.js"); +const SHARED_VERSION_URL = repositoryFileUrl("shared/version.js"); const SHARED_REDIS_MOCK = ` -export class WatchError extends Error {} - function ensureState() { if (!globalThis.__authMockState) { throw new Error("auth-index harness: globalThis.__authMockState not initialized"); @@ -41,7 +44,16 @@ function bumpKeyVersion(key) { s.keyVersions.set(key, (s.keyVersions.get(key) || 0) + 1); } +function expireStringIfNeeded(key) { + const s = ensureState(); + const expiresAt = s.expirations.get(key); + if (expiresAt == null || expiresAt > Date.now()) return; + s.expirations.delete(key); + if (s.strings.delete(key)) bumpKeyVersion(key); +} + function keyVersion(key) { + expireStringIfNeeded(key); const s = ensureState(); return s.keyVersions.get(key) || 0; } @@ -53,6 +65,7 @@ export class RedisClient { } async get(key) { const s = ensureState(); + expireStringIfNeeded(key); if (s.getThrows && s.getThrows.has(key)) { const err = new Error("forced get throw on " + key); recordCommand("GET", false, err.message); @@ -88,13 +101,16 @@ export class RedisClient { } async del(key) { const s = ensureState(); + expireStringIfNeeded(key); recordCommand("DEL", true); const had = s.strings.delete(key) || s.hashes.delete(key); + s.expirations.delete(key); if (had) bumpKeyVersion(key); return had ? 1 : 0; } async delIfEq(key, value) { const s = ensureState(); + expireStringIfNeeded(key); if (s.delIfEqThrows && s.delIfEqThrows.has(key)) { const err = new Error("forced delIfEq throw on " + key); recordCommand("DELIFEQ", false, err.message); @@ -103,9 +119,33 @@ export class RedisClient { recordCommand("DELIFEQ", true); if (s.strings.get(key) !== value) return 0; s.strings.delete(key); + s.expirations.delete(key); bumpKeyVersion(key); return 1; } + async set(key, value, options = {}) { + const s = ensureState(); + expireStringIfNeeded(key); + if (options.nx && s.strings.has(key)) { + recordCommand("SET", true); + return null; + } + s.strings.set(key, value); + if (typeof options.ttl === "number") { + s.expirations.set(key, Date.now() + options.ttl * 1000); + } else { + s.expirations.delete(key); + } + bumpKeyVersion(key); + try { + if (s.afterSetAppliedBeforeReply) s.afterSetAppliedBeforeReply(key, value, options); + } catch (err) { + recordCommand("SET", false, err instanceof Error ? err.message : String(err)); + throw err; + } + recordCommand("SET", true); + return "OK"; + } async scan(_cursor, pattern, _count) { const s = ensureState(); recordCommand("SCAN", true); @@ -135,7 +175,9 @@ export class RedisClient { for (const cmd of cmds) { const [op, key, ...rest] = cmd; if (op === "DEL") { + expireStringIfNeeded(key); const had = s.strings.delete(key) || s.hashes.delete(key); + s.expirations.delete(key); if (had) bumpKeyVersion(key); out.push(had ? 1 : 0); recordCommand("DEL", true); @@ -149,11 +191,19 @@ export class RedisClient { out.push(rest.length / 2); recordCommand("HSET", true); } else if (op === "SET") { - const options = rest.slice(1).map((arg) => String(arg).toUpperCase()); - if (options.includes("NX") && s.strings.has(key)) { + expireStringIfNeeded(key); + const options = rest.slice(1).map((arg) => String(arg)); + const upperOptions = options.map((arg) => arg.toUpperCase()); + if (upperOptions.includes("NX") && s.strings.has(key)) { out.push(null); } else { s.strings.set(key, rest[0]); + const exIndex = upperOptions.indexOf("EX"); + if (exIndex >= 0) { + s.expirations.set(key, Date.now() + Number(options[exIndex + 1]) * 1000); + } else { + s.expirations.delete(key); + } bumpKeyVersion(key); out.push("OK"); } @@ -162,6 +212,12 @@ export class RedisClient { throw new Error("multiExec mock: unsupported " + op); } } + try { + if (s.afterMultiExecAppliedBeforeReply) s.afterMultiExecAppliedBeforeReply(cmds); + } catch (err) { + recordCommand("MULTI_EXEC", false, err instanceof Error ? err.message : String(err)); + throw err; + } return out; } async session(fn) { @@ -176,6 +232,7 @@ export class RedisClient { async get(key) { return self.get(key); }, async del(key) { return self.del(key); }, async delIfEq(key, value) { return self.delIfEq(key, value); }, + async set(key, value, options) { return self.set(key, value, options); }, async hGet(key, field) { return self.hGet(key, field); }, async hGetAll(key) { return self.hGetAll(key); }, async hGetAllMany(keys) { return self.hGetAllMany(keys); }, @@ -301,7 +358,7 @@ export function resolveDelegatedIssueTemplate(templateId, configured = createDel const authLib = await import(authLibUrl); // Mocks have no module-level state; cache them across loads. - const sharedRedisUrl = moduleDataUrl(SHARED_REDIS_MOCK); + const sharedRedisUrl = sharedRedisStubUrl(SHARED_REDIS_MOCK); const sharedObservabilityUrl = moduleDataUrl(SHARED_OBSERVABILITY_MOCK); const authRuntimeUrl = freshRepositoryModuleDataUrl("auth/runtime.js", [ @@ -309,6 +366,7 @@ export function resolveDelegatedIssueTemplate(templateId, configured = createDel [/from "shared-observability"/g, `from ${JSON.stringify(sharedObservabilityUrl)}`], [/from "auth-lib"/g, `from ${JSON.stringify(authLibUrl)}`], [/from "shared-auth-roles"/g, `from ${JSON.stringify(sharedAuthRolesUrl)}`], + [/from "shared-optimistic-retry"/g, `from ${JSON.stringify(repositoryFileUrl("shared/optimistic-retry.js"))}`], ]); const indexUrl = freshRepositoryModuleDataUrl("auth/index.js", [ @@ -316,6 +374,8 @@ export function resolveDelegatedIssueTemplate(templateId, configured = createDel [/from "auth-lib"/g, `from ${JSON.stringify(authLibUrl)}`], [/from "auth-runtime"/g, `from ${JSON.stringify(authRuntimeUrl)}`], [/from "shared-auth-roles"/g, `from ${JSON.stringify(sharedAuthRolesUrl)}`], + [/from "shared-redis-lock"/g, `from ${JSON.stringify(SHARED_REDIS_LOCK_URL)}`], + [/from "shared-version"/g, `from ${JSON.stringify(SHARED_VERSION_URL)}`], ]); const indexMod = await import(indexUrl); @@ -329,6 +389,7 @@ export function resolveDelegatedIssueTemplate(templateId, configured = createDel export function resetAuthMockState() { /** @type {any} */ (globalThis).__authMockState = { strings: new Map(), + expirations: new Map(), hashes: new Map(), sets: new Map(), getThrows: new Set(), @@ -336,6 +397,8 @@ export function resetAuthMockState() { delIfEqThrows: new Set(), scanPages: [], keyVersions: new Map(), + afterSetAppliedBeforeReply: null, + afterMultiExecAppliedBeforeReply: null, beforeMultiExec: null, onCommand: null, commands: [], diff --git a/tests/helpers/load-control-lib.js b/tests/helpers/load-control-lib.js index d66bc81..5e937b2 100644 --- a/tests/helpers/load-control-lib.js +++ b/tests/helpers/load-control-lib.js @@ -41,6 +41,7 @@ export async function compileControlGraph(opts = {}) { const libUrl = freshRepositoryModuleDataUrl("control/lib.js", [ [/from "shared-ns-pattern"/g, `from ${JSON.stringify(SHARED_NS_URL)}`], [/from "shared-auth-roles"/g, `from ${JSON.stringify(sharedAuthRolesUrl)}`], + [/from "shared-errors"/g, `from ${JSON.stringify(SHARED_ERRORS_URL)}`], [/from "shared-version"/g, `from ${JSON.stringify(SHARED_VERSION_URL)}`], ]); @@ -78,6 +79,7 @@ export async function compileControlGraph(opts = {}) { ...importSpecifierReplacements({ "shared-ns-pattern": SHARED_NS_URL, "shared-workerd-compat-flags": SHARED_WORKERD_COMPAT_FLAGS_URL, + "control-lib": libUrl, }), [/from "control-bindings"/g, `from ${JSON.stringify(bindingsUrl)}`], [/from "wdl-package-json-source"/g, `from ${JSON.stringify(packageJsonSourceUrl)}`], diff --git a/tests/helpers/load-control-routing.js b/tests/helpers/load-control-routing.js index 2ad9100..ac4aaf0 100644 --- a/tests/helpers/load-control-routing.js +++ b/tests/helpers/load-control-routing.js @@ -6,6 +6,7 @@ import { } from "./load-shared-module.js"; import { controlSharedStubUrl } from "./control-shared-stub.js"; import { compileControlGraph } from "./load-control-lib.js"; +import { sharedRedisStubUrl } from "./mocks/fake-redis.js"; const { sharedNsUrl, @@ -21,10 +22,8 @@ const { routePlanUrl, } = await compileControlGraph(); -const controlSharedUrl = controlSharedStubUrl(` -export const DECLARED_HOSTS_KEY = "declared-hosts"; -export const HOST_DECLARATIONS_PREFIX = "host-declarations:"; -`); +const controlSharedUrl = controlSharedStubUrl(); +const sharedRedisUrl = sharedRedisStubUrl(); export const CONTROL_ROUTING_TEST_URL = moduleDataUrl(applyModuleReplacements(readRepositoryFile("control/routing.js"), [ ...importSpecifierReplacements({ @@ -39,6 +38,7 @@ export const CONTROL_ROUTING_TEST_URL = moduleDataUrl(applyModuleReplacements(re "shared-ns-pattern": sharedNsUrl, "shared-auth-roles": sharedAuthRolesUrl, "shared-queue-keys": sharedQueueKeysUrl, + "shared-redis": sharedRedisUrl, "control-routing-route-plan": routePlanUrl, }), ])); diff --git a/tests/helpers/load-d1-owner-client.js b/tests/helpers/load-d1-owner-client.js index d622f5f..82070bb 100644 --- a/tests/helpers/load-d1-owner-client.js +++ b/tests/helpers/load-d1-owner-client.js @@ -42,6 +42,8 @@ const d1OwnerClientModule = await importOwnerClientModule("d1-runtime/owner-clie "d1-runtime-owner-registry": ownerRegistryUrl, "shared-internal-auth": ownerHarness.internalAuthUrl, "shared-owner-forwarder": ownerHarness.ownerForwarderUrl, + "shared-owner-endpoint": ownerHarness.ownerEndpointUrl, + "shared-owner-lease": ownerHarness.ownerLeaseUrl, "d1-runtime-state": ownerHarness.stateUrl, }); diff --git a/tests/helpers/load-d1-owner-registry.js b/tests/helpers/load-d1-owner-registry.js index 63b2a80..928d687 100644 --- a/tests/helpers/load-d1-owner-registry.js +++ b/tests/helpers/load-d1-owner-registry.js @@ -7,7 +7,11 @@ import { repositoryModuleDataUrl, sharedModuleDataUrl, } from "./load-shared-module.js"; -import { createFakeRedisState, resetFakeRedisState } from "./mocks/fake-redis.js"; +import { + createFakeRedisState, + resetFakeRedisState, + sharedRedisStubUrl, +} from "./mocks/fake-redis.js"; import { delay } from "./timing.js"; const FAKE_REDIS_URL = repositoryFileUrl("tests/helpers/mocks/fake-redis.js"); @@ -44,7 +48,7 @@ export function resetD1OwnerRegistryTestState() { D1_OWNER_REGISTRY_TEST_STATE.forgottenStorageSizes = []; D1_OWNER_REGISTRY_TEST_STATE.pendingQueries = 0; D1_OWNER_REGISTRY_TEST_STATE.draining = false; - D1_OWNER_REGISTRY_TEST_STATE.taskIdentity = { taskId: "task-a", endpoint: "task-a:8787" }; + D1_OWNER_REGISTRY_TEST_STATE.taskIdentity = { taskId: "task-a", endpoint: "d1-runtime-a:8787" }; D1_OWNER_REGISTRY_TEST_STATE.onWatchExecFailure = null; D1_OWNER_REGISTRY_TEST_STATE.logEntries = []; D1_OWNER_REGISTRY_TEST_STATE.metricIncrements = []; @@ -87,16 +91,7 @@ export async function resolveTaskIdentity() { } `); -const redisUrl = moduleDataUrl(` -const D1_TEST_STATE = /** @type {any} */ (globalThis).__d1OwnerRegistryTestState; -export { FakeRedisWatchError as WatchError } from ${JSON.stringify(FAKE_REDIS_URL)}; -export function decodeBulk(value) { - if (value == null) return value; - if (typeof value === "string") return value; - if (value instanceof Uint8Array) return new TextDecoder().decode(value); - return String(value); -} -`); +const redisUrl = sharedRedisStubUrl(); const redisClientUrl = moduleDataUrl(` import { createFakeRedisClient, createFakeRedisSession } from ${JSON.stringify(FAKE_REDIS_URL)}; @@ -178,6 +173,7 @@ const src = applyModuleReplacements(readRepositoryFile("d1-runtime/owner-registr [/from "shared-redis";/, `from ${JSON.stringify(redisUrl)};`], [/from "shared-env";/, `from ${JSON.stringify(SHARED_ENV_URL)};`], [/from "shared-errors";/, `from ${JSON.stringify(repositoryFileUrl("shared/errors.js"))};`], + [/from "shared-owner-endpoint";/, `from ${JSON.stringify(repositoryFileUrl("shared/owner-endpoint.js"))};`], [/from "shared-owner-lease";/, `from ${JSON.stringify(SHARED_OWNER_LEASE_URL)};`], [/from "shared-owner-protocol";/, `from ${JSON.stringify(SHARED_OWNER_PROTOCOL_URL)};`], [/from "shared-redis-client";/, `from ${JSON.stringify(redisClientUrl)};`], diff --git a/tests/helpers/load-d1-protocol.js b/tests/helpers/load-d1-protocol.js index 11b7c34..c149ac2 100644 --- a/tests/helpers/load-d1-protocol.js +++ b/tests/helpers/load-d1-protocol.js @@ -5,6 +5,7 @@ const DATA_FIELD_URL = repositoryFileUrl("shared/d1-data-field.js"); const FNV_URL = repositoryFileUrl("shared/fnv1a32.js"); const BOUNDED_BODY_URL = repositoryFileUrl("shared/bounded-body.js"); const SHARED_ERRORS_URL = repositoryFileUrl("shared/errors.js"); +const NS_PATTERN_URL = repositoryFileUrl("shared/ns-pattern.js"); export function d1ProtocolDataUrl() { const queryWireUrl = d1QueryWireDataUrl(); @@ -15,6 +16,7 @@ export function d1ProtocolDataUrl() { `import { fnv1a32CodeUnits } from ${JSON.stringify(FNV_URL)};`], [/from "shared-bounded-body";/g, `from ${JSON.stringify(BOUNDED_BODY_URL)};`], [/from "shared-errors";/g, `from ${JSON.stringify(SHARED_ERRORS_URL)};`], + [/from "shared-ns-pattern";/g, `from ${JSON.stringify(NS_PATTERN_URL)};`], [/from "shared-d1-query-wire";/g, `from ${JSON.stringify(queryWireUrl)};`], ]); } diff --git a/tests/helpers/load-do-host-actor.js b/tests/helpers/load-do-host-actor.js index 65ef04b..7d56a3e 100644 --- a/tests/helpers/load-do-host-actor.js +++ b/tests/helpers/load-do-host-actor.js @@ -4,6 +4,7 @@ import { moduleDataUrl, repositoryFileUrl, } from "./load-shared-module.js"; +import { doProtocolDataUrl } from "./load-do-protocol.js"; import { OBSERVABILITY_NOOP_URL } from "./mocks/observability.js"; /** @@ -18,6 +19,8 @@ import { OBSERVABILITY_NOOP_URL } from "./mocks/observability.js"; * assertCalls: number, * forgottenOwners: string[], * registryError?: unknown, + * registryWait?: Promise, + * registryWaitStarted?: (() => void), * abortReject?: ((reason: unknown) => void), * }} DoHostActorHarnessState */ @@ -38,6 +41,7 @@ const DO_ACTOR_TEST_STATE = { doActorGlobal.__doActorTestState = DO_ACTOR_TEST_STATE; const durableObjectStub = "class DurableObject { constructor(ctx, env) { this.ctx = ctx; this.env = env; } }"; +const productionProtocolUrl = doProtocolDataUrl(); const loadUrl = moduleDataUrl(` export function loadDoWorkerCode() { @@ -50,27 +54,28 @@ export function objectRegistryMember(invoke) { return [invoke.className, invoke.objectName].join(":"); } export async function rememberDoObject(env, invoke) { + if (globalThis.__doActorTestState.registryWait) { + globalThis.__doActorTestState.registryWaitStarted?.(); + await globalThis.__doActorTestState.registryWait; + } if (globalThis.__doActorTestState.registryError) throw globalThis.__doActorTestState.registryError; globalThis.__doActorTestState.remembered.push({ env, invoke }); } `); const protocolUrl = moduleDataUrl(` -export class DoRuntimeError extends Error { - constructor(status, code, message) { - super(message); - this.status = status; - this.code = code; - } -} +export { + DO_OWNERSHIP_CODE, + DO_OWNERSHIP_ERROR_CONTROL_HEADER, + DoRuntimeError, + buildRpcRequest, + doPlatformErrorResponse, +} from ${JSON.stringify(productionProtocolUrl)}; export function buildFacetName(invoke) { return [invoke.className, invoke.objectName].join(":"); } export function buildAlarmRequest() { throw new Error("unexpected alarm request"); } export function buildForwardRequest() { throw new Error("unexpected forward request"); } -export function doErrorResponse(err) { - return Response.json({ error: err?.code || "internal_error", message: err?.message || String(err) }, { status: err?.status || 500 }); -} export function normalizeDoConnectRequest() { throw new Error("unexpected connect normalize"); } export async function readLocalActorInvokeRequest() { throw new Error("unexpected actor request read"); } `); @@ -131,6 +136,7 @@ const source = applyModuleReplacements(readRepositoryFile("do-runtime/actor.js") [/from "do-runtime-state";/g, `from ${JSON.stringify(stateUrl)};`], [/from "shared-errors";/g, `from ${JSON.stringify(repositoryFileUrl("shared/errors.js"))};`], [/from "shared-observability";/g, `from ${JSON.stringify(OBSERVABILITY_NOOP_URL)};`], + [/from "shared-respond";/g, `from ${JSON.stringify(repositoryFileUrl("shared/respond.js"))};`], ]); /** @returns {DoHostActorHarnessState} */ @@ -150,6 +156,8 @@ export function resetDoHostActorHarness() { state.assertCalls = 0; state.forgottenOwners = []; delete state.registryError; + delete state.registryWait; + delete state.registryWaitStarted; delete state.abortReject; } diff --git a/tests/helpers/load-do-owner-client.js b/tests/helpers/load-do-owner-client.js index 5ae6266..b4f3885 100644 --- a/tests/helpers/load-do-owner-client.js +++ b/tests/helpers/load-do-owner-client.js @@ -14,6 +14,7 @@ const doOwnerClientModule = await importOwnerClientModule("do-runtime/owner-clie "do-runtime-protocol": protocolUrl, "shared-internal-auth": ownerHarness.internalAuthUrl, "shared-owner-forwarder": ownerHarness.ownerForwarderUrl, + "shared-owner-lease": ownerHarness.ownerLeaseUrl, "do-runtime-state": ownerHarness.stateUrl, }); diff --git a/tests/helpers/load-do-owner-registry.js b/tests/helpers/load-do-owner-registry.js index 1bc85d2..f472313 100644 --- a/tests/helpers/load-do-owner-registry.js +++ b/tests/helpers/load-do-owner-registry.js @@ -7,7 +7,11 @@ import { repositoryModuleDataUrl, sharedModuleDataUrl, } from "./load-shared-module.js"; -import { createFakeRedisState, resetFakeRedisState } from "./mocks/fake-redis.js"; +import { + createFakeRedisState, + resetFakeRedisState, + sharedRedisStubUrl, +} from "./mocks/fake-redis.js"; const PROTOCOL_URL = doProtocolDataUrl(); const FAKE_REDIS_URL = repositoryFileUrl("tests/helpers/mocks/fake-redis.js"); @@ -31,6 +35,7 @@ const redisState = createFakeRedisState(); * @property {Array<{ level: string, event: string, fields: Record }>} logEntries * @property {number} redisTimeMs * @property {number[]} redisTimeSequence + * @property {(() => void) | null} onExecFailure * @property {boolean} draining * @property {number} inFlightDispatches */ @@ -39,12 +44,13 @@ export const DO_OWNER_REGISTRY_TEST_STATE = { redisState, store: redisState.strings, ownedScopes: new Map(), - taskIdentity: { taskId: "task-a", endpoint: "task-a:8788" }, + taskIdentity: { taskId: "task-a", endpoint: "do-runtime-a:8788" }, watchedKeys: redisState.watched, metricIncrements: [], logEntries: [], redisTimeMs: Date.now(), redisTimeSequence: [], + onExecFailure: null, draining: false, inFlightDispatches: 0, }; @@ -57,11 +63,12 @@ export function resetDoOwnerRegistryTestState() { const testState = DO_OWNER_REGISTRY_TEST_STATE; resetFakeRedisState(testState.redisState); testState.ownedScopes.clear(); - testState.taskIdentity = { taskId: "task-a", endpoint: "task-a:8788" }; + testState.taskIdentity = { taskId: "task-a", endpoint: "do-runtime-a:8788" }; testState.metricIncrements = []; testState.logEntries = []; testState.redisTimeMs = Date.now(); testState.redisTimeSequence = []; + testState.onExecFailure = null; testState.draining = false; testState.inFlightDispatches = 0; } @@ -97,14 +104,7 @@ export async function resolveTaskIdentity() { } `); -const sharedRedisUrl = moduleDataUrl(` -export { FakeRedisWatchError as WatchError } from ${JSON.stringify(FAKE_REDIS_URL)}; -export function decodeBulk(value) { - if (value == null || typeof value === "string") return value; - if (value instanceof Uint8Array) return new TextDecoder().decode(value); - return String(value); -} -`); +const sharedRedisUrl = sharedRedisStubUrl(); const redisUrl = moduleDataUrl(` import { createFakeRedisClient } from ${JSON.stringify(FAKE_REDIS_URL)}; @@ -117,6 +117,7 @@ export function createRedisClient() { if (Array.isArray(sequence) && sequence.length) return sequence.shift(); return testState().redisTimeMs; }, + onExecFailure: () => testState().onExecFailure?.(), }); } `); @@ -132,6 +133,8 @@ const src = applyModuleReplacements(readRepositoryFile("do-runtime/owner-registr [/from "shared-version";/, `from ${JSON.stringify(SHARED_VERSION_URL)};`], [/from "do-runtime-state";/, `from ${JSON.stringify(stateUrl)};`], [/from "shared-errors";/, `from ${JSON.stringify(repositoryFileUrl("shared/errors.js"))};`], + [/from "shared-ns-pattern";/, `from ${JSON.stringify(repositoryFileUrl("shared/ns-pattern.js"))};`], + [/from "shared-owner-endpoint";/, `from ${JSON.stringify(repositoryFileUrl("shared/owner-endpoint.js"))};`], ]); const registry = await import(moduleDataUrl(src)); @@ -143,6 +146,7 @@ export const { ownerGenerationKeyOf, ownerLeaseGuardMs, ownerKeyOf, + parseOwner, renewOwnedScopes, releaseOwner, resolveDoOwner, diff --git a/tests/helpers/load-do-protocol.js b/tests/helpers/load-do-protocol.js index 1ef16ee..dccfe68 100644 --- a/tests/helpers/load-do-protocol.js +++ b/tests/helpers/load-do-protocol.js @@ -8,8 +8,9 @@ const SHARED_FNV_URL = repositoryFileUrl("shared/fnv1a32.js"); const SHARED_WORKER_ID_URL = repositoryFileUrl("shared/worker-id.js"); const SHARED_BOUNDED_BODY_URL = repositoryFileUrl("shared/bounded-body.js"); const SHARED_INTERNAL_AUTH_URL = repositoryFileUrl("shared/internal-auth.js"); +const SHARED_NS_PATTERN_URL = repositoryFileUrl("shared/ns-pattern.js"); const SHARED_RESPOND_URL = repositoryFileUrl("shared/respond.js"); -const SHARED_WORKERD_COMPAT_FLAGS_URL = repositoryFileUrl("shared/workerd-compat-flags.js"); +const SHARED_VERSION_URL = repositoryFileUrl("shared/version.js"); const DO_WIRE_GRAMMAR_URL = repositoryFileUrl("do-runtime/protocol/wire-grammar.js"); const DO_ERRORS_URL = repositoryModuleDataUrl("do-runtime/protocol/errors.js", [ [/from "shared-respond";/g, `from ${JSON.stringify(SHARED_RESPOND_URL)};`], @@ -27,9 +28,10 @@ export function doProtocolDataUrl() { "do-runtime-protocol-errors": DO_ERRORS_URL, "do-runtime-protocol-identity": DO_IDENTITY_URL, "shared-worker-id": SHARED_WORKER_ID_URL, - "shared-workerd-compat-flags": SHARED_WORKERD_COMPAT_FLAGS_URL, "shared-bounded-body": SHARED_BOUNDED_BODY_URL, "shared-internal-auth": SHARED_INTERNAL_AUTH_URL, + "shared-ns-pattern": SHARED_NS_PATTERN_URL, + "shared-version": SHARED_VERSION_URL, }), ]); } diff --git a/tests/helpers/load-owner-client-harness.js b/tests/helpers/load-owner-client-harness.js index 1e6c64b..a38a5ba 100644 --- a/tests/helpers/load-owner-client-harness.js +++ b/tests/helpers/load-owner-client-harness.js @@ -4,6 +4,7 @@ import { moduleDataUrl, repositoryFileUrl, repositoryModuleDataUrl, + sharedModuleDataUrl, } from "./load-shared-module.js"; import { sharedInternalAuthUrl } from "./runtime-proxy-stub.js"; @@ -41,9 +42,12 @@ export function log(level, event, fields) { `); const internalAuthUrl = sharedInternalAuthUrl(); const errorsUrl = repositoryFileUrl("shared/errors.js"); + const ownerEndpointUrl = repositoryFileUrl("shared/owner-endpoint.js"); + const ownerLeaseUrl = sharedModuleDataUrl("shared/owner-lease.js"); const ownerForwarderUrl = repositoryModuleDataUrl("shared/owner-forwarder.js", [ [/from "shared-internal-auth";/, `from ${JSON.stringify(internalAuthUrl)};`], [/from "shared-errors";/, `from ${JSON.stringify(errorsUrl)};`], + [/from "shared-owner-endpoint";/, `from ${JSON.stringify(ownerEndpointUrl)};`], ]); return { @@ -52,6 +56,8 @@ export function log(level, event, fields) { errorsUrl, internalAuthUrl, ownerForwarderUrl, + ownerEndpointUrl, + ownerLeaseUrl, reset() { state.fetches = []; state.logs = []; diff --git a/tests/helpers/load-runtime-dispatch.js b/tests/helpers/load-runtime-dispatch.js index adf5154..2e5d97b 100644 --- a/tests/helpers/load-runtime-dispatch.js +++ b/tests/helpers/load-runtime-dispatch.js @@ -41,6 +41,7 @@ export async function loadRuntimeDispatch() { ]); const tailForwarderUrl = repositoryModuleDataUrl("runtime/tail-forwarder.js", [ [/from "runtime-bindings-proxy";/, `from ${JSON.stringify(PROXY_BINDING_URL)};`], + [/from "shared-errors";/, `from ${JSON.stringify(SHARED_ERRORS_URL)};`], [/from "shared-internal-auth";/, `from ${JSON.stringify(SHARED_INTERNAL_AUTH_URL)};`], ]); diff --git a/tests/helpers/load-runtime-r2-binding.js b/tests/helpers/load-runtime-r2-binding.js index 31ff661..090add4 100644 --- a/tests/helpers/load-runtime-r2-binding.js +++ b/tests/helpers/load-runtime-r2-binding.js @@ -13,6 +13,8 @@ const R2_UTILS_URL = repositoryFileUrl("runtime/r2-utils.js"); const PROXY_BINDING_URL = runtimeProxyBindingStubUrl(); const SHARED_S3_XML_URL = repositoryFileUrl("shared/s3-xml.js"); const SHARED_RESPOND_URL = repositoryFileUrl("shared/respond.js"); +const SHARED_BASE64_URL = repositoryFileUrl("shared/base64.js"); +const SHARED_S3_RETRY_URL = repositoryFileUrl("shared/s3-retry.js"); const R2_METADATA_URL = repositoryModuleDataUrl("runtime/bindings/r2/metadata.js", [ [/from "runtime-r2-utils";/, `from ${JSON.stringify(R2_UTILS_URL)};`], ]); @@ -51,6 +53,8 @@ const mod = await importRepositoryModule("runtime/bindings/r2.js", [ [/from "runtime-bindings-r2-metadata";/, `from ${JSON.stringify(R2_METADATA_URL)};`], [/from "runtime-bindings-r2-xml";/, `from ${JSON.stringify(R2_XML_URL)};`], [/from "shared-respond";/, `from ${JSON.stringify(SHARED_RESPOND_URL)};`], + [/from "shared-base64";/, `from ${JSON.stringify(SHARED_BASE64_URL)};`], + [/from "shared-s3-retry";/, `from ${JSON.stringify(SHARED_S3_RETRY_URL)};`], ]); export const { R2Bucket } = mod; diff --git a/tests/helpers/load-shared-module.js b/tests/helpers/load-shared-module.js index 949dc48..66a8914 100644 --- a/tests/helpers/load-shared-module.js +++ b/tests/helpers/load-shared-module.js @@ -10,7 +10,13 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url)); const REPO_ROOT = path.resolve(__dirname, "../.."); const SHARED_ENV_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/env.js")).href; const SHARED_ERRORS_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/errors.js")).href; +const SHARED_BASE64_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/base64.js")).href; +const SHARED_NS_PATTERN_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/ns-pattern.js")).href; +const SHARED_VERSION_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/version.js")).href; const SHARED_HEX_URL = pathToFileURL(path.resolve(REPO_ROOT, "shared/hex.js")).href; +const SHARED_OPTIMISTIC_RETRY_URL = pathToFileURL( + path.resolve(REPO_ROOT, "shared/optimistic-retry.js") +).href; const SHARED_WORKERD_COMPAT_FLAGS_URL = pathToFileURL( path.resolve(REPO_ROOT, "shared/workerd-compat-flags.js") ).href; @@ -87,6 +93,9 @@ export function repositoryModuleDataUrl(relativePath, replacements = []) { export function runtimeLibModuleDataUrl() { return repositoryModuleDataUrl("runtime/lib.js", importSpecifierReplacements({ + "shared-base64": SHARED_BASE64_URL, + "shared-ns-pattern": SHARED_NS_PATTERN_URL, + "shared-version": SHARED_VERSION_URL, "shared-workerd-compat-flags": SHARED_WORKERD_COMPAT_FLAGS_URL, })); } @@ -123,6 +132,7 @@ export async function importRepositoryModuleFresh(relativePath, replacements = [ export function sharedModuleDataUrl(relativePath) { const src = applyModuleReplacements(readRepositoryFile(relativePath), [ [/from "shared-env";?/g, `from ${JSON.stringify(SHARED_ENV_URL)};`], + [/from "shared-optimistic-retry";?/g, `from ${JSON.stringify(SHARED_OPTIMISTIC_RETRY_URL)};`], [/from "\.\/errors\.js";?/g, `from ${JSON.stringify(SHARED_ERRORS_URL)};`], [/from "\.\/hex\.js";?/g, `from ${JSON.stringify(SHARED_HEX_URL)};`], ]); diff --git a/tests/helpers/mock-global.js b/tests/helpers/mock-global.js index a324449..162ae3e 100644 --- a/tests/helpers/mock-global.js +++ b/tests/helpers/mock-global.js @@ -81,3 +81,26 @@ export async function withMockedProperty(target, name, mockImpl, callback) { restore(); } } + +/** + * Temporarily replaces an accessor or other descriptor for one async scope. + * + * @template {object} T + * @template {keyof T} K + * @template {() => unknown | Promise} TCallback + * @param {T} target + * @param {K} name + * @param {PropertyDescriptor} descriptor + * @param {TCallback} callback + * @returns {Promise>>} + */ +export async function withMockedPropertyDescriptor(target, name, descriptor, callback) { + const originalDescriptor = Object.getOwnPropertyDescriptor(target, name); + Object.defineProperty(target, name, descriptor); + try { + return /** @type {Awaited>} */ (await callback()); + } finally { + if (originalDescriptor) Object.defineProperty(target, name, originalDescriptor); + else delete target[name]; + } +} diff --git a/tests/helpers/mocks/fake-redis.js b/tests/helpers/mocks/fake-redis.js index 7ecab29..bab6d4d 100644 --- a/tests/helpers/mocks/fake-redis.js +++ b/tests/helpers/mocks/fake-redis.js @@ -1,3 +1,16 @@ +import { + moduleDataUrl, + repositoryFileUrl, + repositoryModuleDataUrl, +} from "../load-shared-module.js"; + +const SHARED_OBSERVABILITY_URL = repositoryFileUrl("shared/observability.js"); +const SHARED_ERRORS_URL = repositoryFileUrl("shared/errors.js"); +const SHARED_REDIS_RESP_URL = repositoryModuleDataUrl("shared/redis-resp.js", [ + [/from "shared-observability";/, `from ${JSON.stringify(SHARED_OBSERVABILITY_URL)};`], + [/from "\.\/errors\.js";/, `from ${JSON.stringify(SHARED_ERRORS_URL)};`], +]); + /** @typedef {Map} FakeRedisStrings */ /** @typedef {Map>} FakeRedisHashes */ /** @typedef {Map>} FakeRedisSets */ @@ -26,6 +39,20 @@ export class FakeRedisWatchError extends Error { } } +/** + * Canonical `shared-redis` test surface. Callers may append only the + * state-bound exports their module graph needs. + * @param {string} [extraSource] + */ +export function sharedRedisStubUrl(extraSource = "") { + return moduleDataUrl(` +import { FakeRedisWatchError as WatchError } from ${JSON.stringify(import.meta.url)}; +import { decodeBulk } from ${JSON.stringify(SHARED_REDIS_RESP_URL)}; +export { WatchError, decodeBulk }; +${extraSource} +`); +} + /** @returns {FakeRedisState} */ export function createFakeRedisState() { return { @@ -287,6 +314,18 @@ export function createFakeRedisSession(state, options = {}) { async getWithTime(key) { return { value: await this.get(key), nowMs: await this.time() }; }, + /** @param {string[]} keys */ + async getManyWithTime(keys) { + if (keys.length === 0) throw new Error("getManyWithTime requires at least one key"); + state.commands.push(["getManyWithTime", [...keys]]); + return { + values: keys.map((key) => { + expireIfNeeded(state, key, options); + return state.strings.get(key) ?? null; + }), + nowMs: await this.time(), + }; + }, async time() { return currentFakeRedisTimeMs(state, options); }, diff --git a/tests/helpers/mocks/observability.js b/tests/helpers/mocks/observability.js index 2dcf611..8e5406b 100644 --- a/tests/helpers/mocks/observability.js +++ b/tests/helpers/mocks/observability.js @@ -1,9 +1,15 @@ // Tests that drive logger or metrics spies inline their own variant — the // spy shape is per-test. -import { moduleDataUrl } from "../load-shared-module.js"; +import { moduleDataUrl, repositoryFileUrl } from "../load-shared-module.js"; + +const SHARED_OBSERVABILITY_URL = repositoryFileUrl("shared/observability.js"); const OBSERVABILITY_NOOP_SOURCE = String.raw` +import { formatError, sanitizeRequestId } from ${JSON.stringify(SHARED_OBSERVABILITY_URL)}; + +export { formatError, sanitizeRequestId }; + export class MetricsRegistry { increment() {} observe() {} @@ -15,19 +21,6 @@ export function createLogger() { export function createLogLevelBinder() { return function bindLogLevel() {}; } -export function formatError(err) { - if (!err) return { error_message: "Unknown error" }; - if (err instanceof Error) { - const out = { error_name: err.name, error_message: err.message }; - if (typeof /** @type {any} */ (err).code === "string") { - out.error_code = /** @type {any} */ (err).code; - } else if (typeof /** @type {any} */ (err).reason === "string") { - out.error_code = /** @type {any} */ (err).reason; - } - return out; - } - return { error_message: String(err) }; -} export function recordRedisCommand() {} export function recordRequestComplete() {} export function ensureRequestId(headersLike) { @@ -37,18 +30,6 @@ export function ensureRequestId(headersLike) { : headersLike["x-request-id"]; return sanitizeRequestId(raw) || generateRequestId(); } -export function sanitizeRequestId(raw) { - if (Array.isArray(raw)) raw = raw[0]; - if (typeof raw !== "string") return null; - const first = raw.split(",")[0].trim(); - if (!first || first.length > 128) return null; - if ([...first].some((ch) => ch.trim() === "" || ch === '"' || ch === "\\")) return null; - for (let i = 0; i < first.length; i++) { - const code = first.charCodeAt(i); - if (code < 0x20 || code === 0x7f || (code >= 0x80 && code <= 0x9f)) return null; - } - return first; -} export function generateRequestId() { return "rid"; } diff --git a/tests/helpers/runtime-injection-sources.js b/tests/helpers/runtime-injection-sources.js index 633c8ad..6b095e2 100644 --- a/tests/helpers/runtime-injection-sources.js +++ b/tests/helpers/runtime-injection-sources.js @@ -15,7 +15,7 @@ const REAL_RUNTIME_INJECTION_SOURCE_PATHS = Object.freeze({ r2UtilsSource: "runtime/r2-utils.js", doClientSource: "runtime/do-client.js", doTransportSource: "runtime/_wdl-do-transport.js", - ownerEndpointSource: "runtime/_wdl-owner-endpoint.js", + ownerEndpointSource: "shared/owner-endpoint.js", ownerHintCacheSource: "runtime/_wdl-owner-hint-cache.js", requestIdSource: "runtime/_wdl-request-id.js", workflowsClientSource: "runtime/workflows-client.js", @@ -61,7 +61,7 @@ export const STUB_RUNTIME_INJECTION_SOURCES = Object.freeze({ ownerHintCacheSource: "export function createOwnerHintCache() { return {}; }", requestIdSource: - "export function requestIdFromOptions() { return null; }", + "export function requestIdFromOptions() { return null; } export function sanitizeRequestId() { return null; }", workflowsClientSource: "export class Workflow { constructor(metadata) { this.metadata = metadata; } }", }); diff --git a/tests/helpers/style-contract-scanner.js b/tests/helpers/style-contract-scanner.js index 950dfc5..1fb0d0b 100644 --- a/tests/helpers/style-contract-scanner.js +++ b/tests/helpers/style-contract-scanner.js @@ -1,6 +1,7 @@ import assert from "node:assert/strict"; import { readdirSync } from "node:fs"; import path from "node:path"; +import { isAlias, isSeq, parseAllDocuments, parseDocument, visit } from "yaml"; import { readRepoFile, repoPath } from "./source-scan.js"; @@ -52,6 +53,49 @@ export function testSourceFiles(dir) { return out; } +/** + * @param {string} file + * @param {{ merge?: boolean }} [options] + */ +export function yamlDocument(file, options = {}) { + const document = parseDocument(readRepoFile(file), { merge: options.merge === true }); + assert.equal(document.errors.length, 0, `${file} must parse as YAML`); + return document; +} + +/** @param {string} file */ +export function yamlDocuments(file) { + const documents = parseAllDocuments(readRepoFile(file)); + for (const document of documents) { + assert.equal(document.errors.length, 0, `${file} must parse as YAML`); + } + return documents; +} + +/** + * @param {import("yaml").Document} document + * @param {(string | number)[]} pathParts + * @returns {string[]} + */ +export function yamlMergeAliases(document, pathParts) { + const merge = document.getIn([...pathParts, "<<"], true); + if (isAlias(merge)) return [merge.source]; + assert.ok(isSeq(merge), `YAML path ${pathParts.join(".")} must contain a merge alias`); + assert.ok(merge.items.every(isAlias), `YAML path ${pathParts.join(".")} merge entries must be aliases`); + return merge.items.map((item) => /** @type {import("yaml").Alias} */ (item).source); +} + +/** @param {import("yaml").Document} document @param {string} source */ +export function yamlAliasCount(document, source) { + let count = 0; + visit(document, { + Alias(_key, alias) { + if (alias.source === source) count += 1; + }, + }); + return count; +} + /** @param {Set} [exempt] */ export function scannedTestFiles(exempt = new Set()) { const scanned = [ @@ -153,16 +197,106 @@ export function extractRegex(file, name) { } /** - * @param {string} service - * @param {string} anchor + * @param {string} source + * @param {number} openIndex + * @param {string} open + * @param {string} close */ -export function serviceAnchorRegex(service, anchor) { - return new RegExp(`\\n ${RegExp.escape(service)}:\\n <<: \\*${RegExp.escape(anchor)}\\b`); +function findClosingDelimiter(source, openIndex, open, close) { + let depth = 0; + let quote = ""; + for (let index = openIndex; index < source.length; index += 1) { + const char = source[index]; + if (quote) { + if (char === "\\") index += 1; + else if (char === quote) quote = ""; + continue; + } + if (char === '"') { + quote = char; + } else if (char === open) { + depth += 1; + } else if (char === close) { + depth -= 1; + if (depth === 0) return index; + } + } + return -1; } -/** @param {string} service */ -export function d1RuntimeServiceRegex(service) { - return new RegExp(`\\n ${RegExp.escape(service)}:\\n(?: profiles: \\["d1-multi"\\]\\n)? <<: \\*d1-runtime-service\\b`); +/** @param {string} source */ +export function stripCapnpComments(source) { + let out = ""; + let quoted = false; + for (let index = 0; index < source.length; index += 1) { + const char = source[index]; + if (quoted) { + out += char; + if (char === "\\") { + index += 1; + out += source[index] ?? ""; + } else if (char === '"') { + quoted = false; + } + continue; + } + if (char === '"') { + quoted = true; + out += char; + continue; + } + if (char !== "#") { + out += char; + continue; + } + while (index < source.length && source[index] !== "\n") { + out += " "; + index += 1; + } + if (index < source.length) out += "\n"; + } + return out; +} + +/** + * Extracts a Cap'n Proto const struct while ignoring comments and formatting. + * @param {string} source + * @param {string} name + */ +export function extractCapnpConstBlock(source, name) { + const clean = stripCapnpComments(source); + const declaration = new RegExp(`const\\s+${RegExp.escape(name)}\\s+:[^=]+=`).exec(clean); + assert.ok(declaration, `Cap'n Proto const ${name} must be present`); + const start = clean.indexOf("(", declaration.index + declaration[0].length); + assert.notEqual(start, -1, `Cap'n Proto const ${name} must contain a struct value`); + const end = findClosingDelimiter(clean, start, "(", ")"); + assert.notEqual(end, -1, `Cap'n Proto const ${name} must have balanced parentheses`); + return clean.slice(start, end + 1); +} + +/** + * Returns normalized top-level tuple entries from a list field in a Cap'n Proto struct. + * @param {string} block + * @param {string} field + */ +export function extractCapnpListEntries(block, field) { + const assignment = new RegExp(`\\b${RegExp.escape(field)}\\s*=\\s*\\[`).exec(block); + assert.ok(assignment, `Cap'n Proto field ${field} must be present`); + const listStart = block.indexOf("[", assignment.index); + const listEnd = findClosingDelimiter(block, listStart, "[", "]"); + assert.notEqual(listEnd, -1, `Cap'n Proto field ${field} must have balanced brackets`); + + const entries = []; + let index = listStart + 1; + while (index < listEnd) { + const start = block.indexOf("(", index); + if (start === -1 || start >= listEnd) break; + const end = findClosingDelimiter(block, start, "(", ")"); + assert.notEqual(end, -1, `Cap'n Proto field ${field} entry must have balanced parentheses`); + entries.push(block.slice(start, end + 1).replace(/\s+/g, " ").trim()); + index = end + 1; + } + return entries; } /** diff --git a/tests/integration/cron-triggers.test.js b/tests/integration/cron-triggers.test.js index e652d49..4f73e48 100644 --- a/tests/integration/cron-triggers.test.js +++ b/tests/integration/cron-triggers.test.js @@ -394,6 +394,40 @@ test("scheduler: stranded ref (slotMs < currentSlot) is advanced without firing assert.equal(res.body, "null", "stranded ref must NOT invoke scheduled()"); }); +test("scheduler: invalid persisted cron ref is removed without firing", async () => { + const ns = uniqueNs("cronbadnext"); + const cron = "0 0 1 1 *"; + const d = await adminPost(`/ns/${ns}/worker/w/deploy`, { + code: SCHEDULED_RECORDER, + crons: [{ cron, timezone: "UTC" }], + }); + assertStatus(d, 201, "invalid persisted cron fixture deploy"); + const p = await adminPost(`/ns/${ns}/worker/w/promote`, { version: d.json.version }); + assertStatus(p, 200, "invalid persisted cron fixture promote"); + + const id = cronId(cron, "UTC"); + const entry = cronHashJsonField(ns, id); + const ref = `${ns}:w:${id}:${entry.gen}`; + redisHSet(`crons:${ns}:w`, { + [id]: JSON.stringify({ ...entry, timezone: "Mars/Olympus" }), + }); + + await waitForCurrentSlotFixtureWindow(); + const currentSlot = Math.floor(Date.now() / 60_000) * 60_000; + redisSAdd(`cron-slot:${currentSlot}`, ref); + composeRestart("scheduler"); + + await waitUntil("scheduler to remove invalid persisted cron ref", async () => { + return !redisSMembers(`cron-slot:${currentSlot}`).includes(ref); + }, { timeoutMs: 15_000, intervalMs: 500 }); + + const res = runtimeInternalGetWithHeaders("/", { + "x-worker-id": gatewayWorkerId(ns, "w", d.json.version), + }); + assert.equal(res.status, 200); + assert.equal(res.body, "null", "invalid persisted cron must NOT invoke scheduled()"); +}); + test("scheduler: ref with mismatched gen is SREM'd from bucket without firing", async () => { const ns = uniqueNs("schstale"); await deployAndPromoteWithCrons(ns, "w", [ diff --git a/tests/integration/delete-api.test.js b/tests/integration/delete-api.test.js index cc2cfc2..d6d041c 100644 --- a/tests/integration/delete-api.test.js +++ b/tests/integration/delete-api.test.js @@ -445,7 +445,9 @@ test("POST /delete fail-closes when active route metadata is malformed", async ( assert.equal(r.status, 500); const body = await responseJson(r); assert.equal(body.error, "corrupt_meta"); - assert.equal(body.stage, "active_meta_routes"); + assert.equal(body.message, "Internal error"); + assert.equal(body.stage, undefined); + assert.equal(body.detail, undefined); assert.ok(redisHGetAll("patterns:bad-route.workers.example")["/api/*"]); assert.equal(redisSIsMember(`ns-hosts:${ns}`, "bad-route.workers.example"), true); @@ -656,9 +658,9 @@ test("POST /delete on a worker whose active bundle has corrupt __meta__ → 500 const body = await responseJson(r); assert.equal(body.error, "corrupt_meta"); assert.equal(body.version, "v1"); - // v1 is both active AND the only retained; which parse path trips - // first is an implementation detail, but it must be one of them. - assert.match(body.stage, /meta_parse$/); + assert.equal(body.message, "Internal error"); + assert.equal(body.stage, undefined); + assert.equal(body.detail, undefined); assert.equal(redisHGet(`routes:${ns}`, "api"), preActive); assert.deepEqual(redisSMembers(`workers:${ns}`), preWorkers); @@ -680,7 +682,9 @@ test("POST /delete on a worker whose retained (non-active) bundle has corrupt __ const body = await responseJson(r); assert.equal(body.error, "corrupt_meta"); assert.equal(body.version, "v1"); - assert.equal(body.stage, "retained_meta_parse"); + assert.equal(body.message, "Internal error"); + assert.equal(body.stage, undefined); + assert.equal(body.detail, undefined); // Active pointer must survive — no partial tear-down. assert.equal(redisHGet(`routes:${ns}`, "api"), "v2"); @@ -726,8 +730,10 @@ test("DELETE /versions/ with corrupt sibling meta during shared-prefix scan assert.equal(r.status, 500); const body = await responseJson(r); assert.equal(body.error, "corrupt_meta"); - assert.equal(body.stage, "sibling_meta_parse"); assert.equal(body.version, "v2"); + assert.equal(body.message, "Internal error"); + assert.equal(body.stage, undefined); + assert.equal(body.detail, undefined); assert.equal(redisExists(`worker:${ns}:w:v:1`), true); assert.equal(queueCleanupIntents().length, 0); diff --git a/tests/integration/durable-objects-alarms.test.js b/tests/integration/durable-objects-alarms.test.js index 76e3313..7ef561f 100644 --- a/tests/integration/durable-objects-alarms.test.js +++ b/tests/integration/durable-objects-alarms.test.js @@ -62,6 +62,54 @@ const DO_BLOCKING_ALARM_WORKER = readFileSync( "utf8" ); +const DO_ACCESSOR_ALARM_WORKER = ` +const instances = new WeakMap(); + +export class AccessorAlarm { + #ctx; + + constructor(ctx) { + this.#ctx = ctx; + instances.set(this, true); + } + + #assertReceiver() { + if (!instances.has(this)) throw new TypeError("invalid AccessorAlarm receiver"); + } + + get fetch() { + this.#assertReceiver(); + return async (request) => { + const url = new URL(request.url); + if (url.pathname === "/schedule") { + await this.#ctx.storage.setAlarm(Date.now() - 1000); + return Response.json({ pending: true }); + } + return Response.json({ + alarms: (await this.#ctx.storage.get("alarms")) ?? 0, + pending: await this.#ctx.storage.getAlarm(), + }); + }; + } + + get alarm() { + this.#assertReceiver(); + return async () => { + const alarms = (await this.#ctx.storage.get("alarms")) ?? 0; + await this.#ctx.storage.put("alarms", alarms + 1); + }; + } +} + +export default { + async fetch(request, env) { + const url = new URL(request.url); + const id = env.ACCESSORS.idFromName(url.searchParams.get("name") || "main"); + return await env.ACCESSORS.get(id).fetch("https://do.internal" + url.pathname); + }, +}; +`; + /** * @param {string} ns * @param {string} worker @@ -252,6 +300,34 @@ test("do-runtime shim supports storage alarms on SQLite-backed Durable Objects", assert.equal(spoofJson.pending, null); }); +test("do-runtime alarm wrapper preserves accessor handler receivers through host proxies", async () => { + const ns = uniqueNs("do-alarm-accessor"); + await deployAndPromote(ns, "alarms", { + mainModule: "worker.js", + modules: { "worker.js": DO_ACCESSOR_ALARM_WORKER }, + bindings: { + ACCESSORS: { type: "do", className: "AccessorAlarm" }, + }, + }); + + const scheduled = await gatewayFetch(ns, "/alarms/schedule?name=alice"); + const scheduledText = await scheduled.text(); + assert.equal(scheduled.status, 200, scheduledText); + assert.deepEqual(responseJson({ body: scheduledText }), { pending: true }); + + const status = await waitForJson( + "accessor-backed DO alarm", + async () => { + const response = await gatewayFetch(ns, "/alarms/status?name=alice"); + const body = await response.text(); + assert.equal(response.status, 200, body); + return responseJson({ body }); + }, + (json) => json.alarms === 1 && json.pending === null + ); + assert.deepEqual(status, { alarms: 1, pending: null }); +}); + test("scheduler replicas: Workflows DO alarm tick delivers a due alarm once", async () => { const ns = uniqueNs("do-alarm-replica"); await deployAndPromote(ns, "alarms", { diff --git a/tests/integration/durable-objects-core.test.js b/tests/integration/durable-objects-core.test.js index b25cfe1..ce8e75c 100644 --- a/tests/integration/durable-objects-core.test.js +++ b/tests/integration/durable-objects-core.test.js @@ -1,3 +1,4 @@ +import { readFileSync } from "node:fs"; import { test } from "node:test"; import assert from "node:assert/strict"; import { @@ -13,9 +14,15 @@ import { DO_RPC_WORKER, DO_WORKER, } from "./helpers/durable-objects.js"; +import { redisDel, redisSetEx } from "./helpers/redis.js"; setupIntegrationSuite(); +const DO_INTRINSIC_GUARD_WORKER = readFileSync( + new URL("../fixtures/do-intrinsic-guard-worker.mjs.txt", import.meta.url), + "utf8" +); + test("worker Durable Object binding routes through do-runtime and preserves object state", async () => { const ns = uniqueNs("do"); await deployAndPromote(ns, "counter", { @@ -57,6 +64,57 @@ test("worker Durable Object binding routes through do-runtime and preserves obje }); }); +test("version-delete lock does not interrupt active Durable Object traffic", async () => { + const ns = uniqueNs("do-version-delete"); + await deployAndPromote(ns, "counter", { + mainModule: "worker.js", + modules: { "worker.js": DO_WORKER }, + bindings: { + COUNTER: { type: "do", className: "Counter" }, + }, + }); + + const lockKey = `worker-delete-lock:${ns}:counter`; + redisSetEx(lockKey, "version:integration-token", 30); + try { + const response = await gatewayFetch(ns, "/counter?name=alice"); + const text = await response.text(); + assert.equal(response.status, 200, text); + assert.deepEqual(responseJson({ body: text }), { + objectId: "alice", + memory: 1, + storage: 1, + body: "from-worker", + }); + } finally { + redisDel(lockKey); + } +}); + +test("tenant top-level intrinsic patches cannot bypass host env wrapping", async () => { + const ns = uniqueNs("do-intrinsic-guard"); + await deployAndPromote(ns, "guard", { + mainModule: "worker.js", + modules: { "worker.js": DO_INTRINSIC_GUARD_WORKER }, + vars: { BYPASS: true }, + bindings: { + ROOM: { type: "do", className: "Room" }, + }, + }); + + const response = await gatewayFetch(ns, "/guard"); + const text = await response.text(); + assert.equal(response.status, 200, text); + assert.deepEqual(responseJson({ body: text }), { + room: "room-ok", + roomFacade: "DurableObjectNamespace", + ownerMetadataVisible: false, + doBackendVisible: false, + ownerNetworkVisible: false, + workflowsBackendVisible: false, + }); +}); + test("Durable Object classes receive ordinary Worker bindings", async () => { const ns = uniqueNs("do-bindings"); await deployAndPromote(ns, "rooms", { @@ -100,6 +158,7 @@ test("Durable Object RPC dispatches class methods with structured args", async ( await deployAndPromote(ns, "rooms", { mainModule: "worker.js", modules: { "worker.js": DO_RPC_WORKER }, + compatibilityFlags: ["no_nodejs_als"], bindings: { ROOM: { type: "do", className: "Room" }, }, @@ -175,4 +234,30 @@ test("Durable Object RPC dispatches class methods with structured args", async ( meta: { role: "peer" }, }, }); + + const requestId = "rid-do-rpc-context"; + const context = await gatewayFetch(ns, "/rooms/request-id?name=alice&to=bob", { + headers: { "x-request-id": requestId }, + }); + const contextText = await context.text(); + assert.equal(context.status, 200, contextText); + assert.deepEqual(responseJson({ body: contextText }), { requestId }); + + const nestedContext = await gatewayFetch( + ns, + "/rooms/nested-request-id?name=alice&to=bob", + { headers: { "x-request-id": requestId } } + ); + const nestedContextText = await nestedContext.text(); + assert.equal(nestedContext.status, 200, nestedContextText); + assert.deepEqual(responseJson({ body: nestedContextText }), { requestId }); + + const reenteredContext = await gatewayFetch( + ns, + "/rooms/reenter-request-id?name=alice&to=bob", + { headers: { "x-request-id": requestId } } + ); + const reenteredContextText = await reenteredContext.text(); + assert.equal(reenteredContext.status, 200, reenteredContextText); + assert.deepEqual(responseJson({ body: reenteredContextText }), { requestId }); }); diff --git a/tests/integration/durable-objects-ownership.test.js b/tests/integration/durable-objects-ownership.test.js index c3d2286..cbff8f6 100644 --- a/tests/integration/durable-objects-ownership.test.js +++ b/tests/integration/durable-objects-ownership.test.js @@ -72,7 +72,7 @@ test("Durable Object takeover preserves committed SQLite state after owner loss" assert.equal(owner.taskId, "do-runtime-a"); const shortLeaseOwner = { ...owner, - leaseExpiresAt: Date.now() + 1000, + leaseExpiresAt: Date.now() + 30_000, }; redisSetDoOwner(ownerKey, shortLeaseOwner); const renew = serviceInternalPost("do-runtime-a", 8788, "/internal/do/renew", {}); @@ -122,6 +122,9 @@ test("Durable Object takeover preserves committed SQLite state after owner loss" storage: 3, body: "from-worker", }); + }, { + // Keep heartbeat renewal outside this explicit-renew assertion window. + renewStartDelayMs: 600_000, }); }); @@ -381,8 +384,8 @@ test("do-runtime replicas forward a sharded object owner scope instead of splitt worker: "counter", doStorageId, className: "Counter", - taskId: "stale-task", - endpoint: "stale-task:8788", + taskId: "missing-owner", + endpoint: "do-runtime-missing:8788", generation: 42, leaseExpiresAt: Date.now() - 1000, }); diff --git a/tests/integration/helpers/durable-objects.js b/tests/integration/helpers/durable-objects.js index c6fada1..6a6a0cb 100644 --- a/tests/integration/helpers/durable-objects.js +++ b/tests/integration/helpers/durable-objects.js @@ -1,6 +1,7 @@ import { readFileSync } from "node:fs"; import assert from "node:assert/strict"; import { fnv1a32CodeUnits } from "../../../shared/fnv1a32.js"; +import { doStorageIdKey } from "../../../shared/version.js"; import { doAlarmJobIdForStorage } from "../../helpers/do-alarm-job-id.js"; import { serviceInternalPost, serviceInternalPostAsync } from "./internal-http.js"; import { @@ -147,7 +148,7 @@ export function redisAddDoAlarmByWorker(ns, worker, jobId) { /** @param {string} ns @param {string} worker */ export function redisGetDoStorageId(ns, worker) { - return redisGetRaw(`worker:do-storage:${ns}:${worker}`) ?? ""; + return redisGetRaw(doStorageIdKey(ns, worker)) ?? ""; } /** diff --git a/tests/integration/helpers/env.js b/tests/integration/helpers/env.js index d982bc2..70cd744 100644 --- a/tests/integration/helpers/env.js +++ b/tests/integration/helpers/env.js @@ -2,7 +2,12 @@ // tests. Imported by nearly every other helper; never imports sibling helpers. import { + ADMIN_HOST_HEADER, + LOCAL_ADMIN_TOKEN, + LOCAL_CONNECT_HOST, ROOT, + localAssetsCdnBase, + localControlUrl, resolveWdlCliBin, } from "../../../scripts/integration-environment.js"; @@ -10,22 +15,22 @@ export { ROOT }; // Control reaches through gateway with Host: admin.test — // gateway's admin-host short-circuit picks it up. Matches docker-compose.yml. -export const GATEWAY_HOST = "localhost"; +export const GATEWAY_HOST = LOCAL_CONNECT_HOST; export const GATEWAY_PORT = Number(process.env.WDL_GATEWAY_HOST_PORT || 8080); -export const S3MOCK_HOST = "localhost"; +export const S3MOCK_HOST = LOCAL_CONNECT_HOST; export const S3MOCK_PORT = Number(process.env.WDL_S3MOCK_HOST_PORT || 19500); -export const ASSETS_CDN_BASE = `http://${S3MOCK_HOST}:${S3MOCK_PORT}/wdl-assets`; -export const ADMIN_HOST_HEADER = "admin.test"; -export const ADMIN_TOKEN = "local-dev-token"; +export const ASSETS_CDN_BASE = localAssetsCdnBase(S3MOCK_PORT); +export { ADMIN_HOST_HEADER }; +export const ADMIN_TOKEN = LOCAL_ADMIN_TOKEN; export const WDL_CLI_BIN = resolveWdlCliBin(); // Integration tests are local-compose only. Force the control env so host // shell credentials (staging/production ADMIN_TOKEN, CONTROL_URL, WDL_NS, etc.) cannot // leak into adminFetch() or CLI child processes and produce misleading 401s. process.env.ADMIN_TOKEN = ADMIN_TOKEN; -export const CONTROL_URL = `http://${ADMIN_HOST_HEADER}:${GATEWAY_PORT}`; +export const CONTROL_URL = localControlUrl(GATEWAY_PORT); process.env.CONTROL_URL = CONTROL_URL; process.env.ASSETS_CDN_BASE = ASSETS_CDN_BASE; // admin.test has no DNS on most machines — aim the socket at localhost. -process.env.CONTROL_CONNECT_HOST = "localhost"; +process.env.CONTROL_CONNECT_HOST = LOCAL_CONNECT_HOST; delete process.env.WDL_NS; delete process.env.CLOUDFLARE_ENV; diff --git a/tests/integration/helpers/misc.js b/tests/integration/helpers/misc.js index 98bcf7a..3707819 100644 --- a/tests/integration/helpers/misc.js +++ b/tests/integration/helpers/misc.js @@ -1,4 +1,5 @@ import { createHash } from "node:crypto"; +import { bundleKey } from "../../../shared/version.js"; import { redisHGetJson } from "./redis.js"; @@ -13,6 +14,6 @@ export function cronId(cron, timezone) { * @param {string} version */ export function readMeta(ns, name, version) { - const key = `worker:${ns}:${name}:v:${version.slice(1)}`; + const key = bundleKey(ns, name, version); return redisHGetJson(key, "__meta__", { label: `${key} __meta__` }); } diff --git a/tests/integration/helpers/stack.js b/tests/integration/helpers/stack.js index 520bc4c..0c5e438 100644 --- a/tests/integration/helpers/stack.js +++ b/tests/integration/helpers/stack.js @@ -182,7 +182,7 @@ export async function waitForScheduler() { redisHSet(consumerKey, { worker: "probe", version: "v1", - max_batch_size: "1000", + max_batch_size: "100", max_batch_timeout_ms: "1000", max_retries: "0", retry_delay_secs: "0", diff --git a/tests/integration/queue-native-dispatch.test.js b/tests/integration/queue-native-dispatch.test.js index 2742001..b8cbca0 100644 --- a/tests/integration/queue-native-dispatch.test.js +++ b/tests/integration/queue-native-dispatch.test.js @@ -145,4 +145,21 @@ test("/_queued: rejects malformed requests with 400", async () => { { queue: "x", messages: [{ id: "m", first_seen_ms: 0, attempts: 0, body_b64: "", content_type: "v8" }] } ); assertStatus(badContentType, 400, "bad content type request"); + + const badBase64 = runtimeDispatchPost( + "/_queued", + { "x-worker-id": workerId }, + { + queue: "x", + messages: [{ + id: "m", + first_seen_ms: 0, + attempts: 0, + body_b64: "%%%", + content_type: "bytes", + }], + } + ); + assertStatus(badBase64, 400, "invalid base64 request"); + assert.equal(responseJson(badBase64).error, "queue_message_decode_failed"); }); diff --git a/tests/integration/service-bindings-rpc.test.js b/tests/integration/service-bindings-rpc.test.js index f5da0fe..9000f17 100644 --- a/tests/integration/service-bindings-rpc.test.js +++ b/tests/integration/service-bindings-rpc.test.js @@ -511,11 +511,9 @@ const CHAIN_INNER_SRC = ` throw err; } async boomPlainNoMessage() { - // eslint-disable-next-line no-throw-literal throw { code: "E_PLAIN", detail: "plain-obj" }; } async boomPlainWithMessage() { - // eslint-disable-next-line no-throw-literal throw { message: "from-inner-plain", code: "E_PLAIN_MSG", detail: "plain-msg" }; } } diff --git a/tests/unit/auth-index.test.js b/tests/unit/auth-index.test.js index 27b5a55..45e92e7 100644 --- a/tests/unit/auth-index.test.js +++ b/tests/unit/auth-index.test.js @@ -826,7 +826,7 @@ test("delegatedIssue: local lock budget is measured from lock acquisition", asyn }); }); -test("delegatedIssue: releases issue lock when SET reply is lost after apply", async () => { +test("delegatedIssue: leaves an uncertain applied lock for TTL expiry after a lost SET reply", async () => { const { auth } = await freshAuth(); const issuer = await auth.issue({ kind: "token-issuer", @@ -836,22 +836,66 @@ test("delegatedIssue: releases issue lock when SET reply is lost after apply", a const state = authMockState(); const lockKey = delegatedIssueLockKey(issuer.tokenId); let lostReplyInjected = false; - state.beforeMultiExec = (/** @type {unknown[][]} */ cmds) => { - if (lostReplyInjected) return; - const lockSet = cmds.find((cmd) => cmd[0] === "SET" && cmd[1] === lockKey); - if (!lockSet) return; + let nowMs = 1_750_000_000_000; + state.afterSetAppliedBeforeReply = (/** @type {string} */ key) => { + if (lostReplyInjected || key !== lockKey) return; lostReplyInjected = true; - state.strings.set(lockKey, lockSet[2]); - state.keyVersions.set(lockKey, (state.keyVersions.get(lockKey) || 0) + 1); throw new Error("lost delegated issue lock SET reply"); }; + await withMockedProperty(Date, "now", () => nowMs, async () => { + await assert.rejects( + () => auth.delegatedIssue({ issuerTokenId: issuer.tokenId, template: "wdl-chat-ns-pool" }), + /lost delegated issue lock SET reply/ + ); + assert.equal(lostReplyInjected, true); + assert.equal(state.strings.has(lockKey), true); + assert.equal(state.expirations.get(lockKey), nowMs + 30_000); + assert.equal(state.commands.at(-1)?.command, "SET"); + assert.equal(state.commands.at(-1)?.ok, false); + assert.equal(state.commands.some((/** @type {any} */ event) => event.command === "DELIFEQ"), false); + + nowMs += 29_999; + await assert.rejects( + () => auth.delegatedIssue({ issuerTokenId: issuer.tokenId, template: "wdl-chat-ns-pool" }), + (err) => /** @type {any} */ (err).reason === "delegated_issue_busy" + ); + + nowMs += 2; + const issued = await auth.delegatedIssue({ + issuerTokenId: issuer.tokenId, + template: "wdl-chat-ns-pool", + }); + assert.equal(issued.kind, "ns"); + assert.equal(state.strings.has(lockKey), false); + assert.equal(state.expirations.has(lockKey), false); + }); +}); + +test("delegatedIssue: leaves its lock for TTL expiry when the final EXEC reply is lost", async () => { + const { auth } = await freshAuth(); + const issuer = await auth.issue({ + kind: "token-issuer", + issueTemplates: ["wdl-chat-ns-pool"], + issuerTokenId: "bootstrap", + }); + const state = authMockState(); + const lockKey = delegatedIssueLockKey(issuer.tokenId); + state.afterMultiExecAppliedBeforeReply = () => { + throw new Error("lost delegated issue EXEC reply"); + }; + await assert.rejects( () => auth.delegatedIssue({ issuerTokenId: issuer.tokenId, template: "wdl-chat-ns-pool" }), - /lost delegated issue lock SET reply/ + /lost delegated issue EXEC reply/ ); - assert.equal(lostReplyInjected, true); - assert.equal(state.strings.has(lockKey), false); + const committed = [...state.hashes.values()].filter((/** @type {any} */ record) => + record.created_by === issuer.tokenId && record.issue_template === "wdl-chat-ns-pool"); + assert.equal(committed.length, 1); + assert.equal(state.strings.has(lockKey), true); + assert.equal(state.commands.at(-1)?.command, "MULTI_EXEC"); + assert.equal(state.commands.at(-1)?.ok, false); + assert.equal(state.commands.some((/** @type {any} */ event) => event.command === "DELIFEQ"), false); }); test("delegatedIssue: issue lock release failure warns without failing issued token", async () => { @@ -922,6 +966,36 @@ test("delegatedIssue: issue lock WATCH prevents writing if the lock changes befo ); }); +test("delegatedIssue: issuer revoke invalidates the final token write", async () => { + const { auth } = await freshAuth(); + const issuer = await auth.issue({ + kind: "token-issuer", + issueTemplates: ["wdl-chat-ns-pool"], + issuerTokenId: "bootstrap", + }); + const state = authMockState(); + const issuerKey = `auth:token:${issuer.tokenId}`; + let revoked = false; + state.beforeMultiExec = (/** @type {unknown[][]} */ cmds) => { + if (revoked || !cmds.some((cmd) => cmd[0] === "HSET" && cmd[1] !== issuerKey)) return; + revoked = true; + state.hashes.get(issuerKey).revoked_at = "2026-07-16T00:00:00.000Z"; + state.keyVersions.set(issuerKey, (state.keyVersions.get(issuerKey) || 0) + 1); + }; + + await assert.rejects( + () => auth.delegatedIssue({ issuerTokenId: issuer.tokenId, template: "wdl-chat-ns-pool" }), + (err) => /** @type {any} */ (err).reason === "delegated_issue_busy" + ); + assert.equal(revoked, true); + assert.equal(state.hashes.get(issuerKey).revoked_at, "2026-07-16T00:00:00.000Z"); + assert.equal( + [...state.hashes.values()].filter((/** @type {any} */ record) => + record.created_by === issuer.tokenId && record.issue_template === "wdl-chat-ns-pool").length, + 0 + ); +}); + test("delegatedIssue: namespace collision retries against namespaces set", async () => { const { auth } = await freshAuth({ delegatedTemplatePatch: TINY_RANDOM_TEMPLATE }); const issuer = await auth.issue({ diff --git a/tests/unit/aws-sigv4.test.js b/tests/unit/aws-sigv4.test.js index 94b93df..88dbc45 100644 --- a/tests/unit/aws-sigv4.test.js +++ b/tests/unit/aws-sigv4.test.js @@ -1,6 +1,7 @@ import assert from "node:assert/strict"; import { test } from "node:test"; +import { SigV4Client as InstalledSigV4Client } from "@wdl-dev/aws-sigv4"; import { SigV4Client } from "../../shared/vendor/aws-sigv4.js"; const ACCESS_KEY_ID = "AKIDEXAMPLE"; @@ -9,6 +10,22 @@ const SESSION_TOKEN = "session-token-example"; const FIXED_DATETIME = "20260616T010203Z"; const S3_ENDPOINT = "https://s3.us-east-1.amazonaws.com"; +/** + * @typedef {{ + * fetch(input: Request | string | URL, init?: import("@wdl-dev/aws-sigv4").SigV4RequestInit): Promise, + * }} TestSigV4Client + * @typedef {(options: import("@wdl-dev/aws-sigv4").SigV4ClientOptions) => TestSigV4Client} TestSigV4ClientFactory + */ + +/** @type {TestSigV4ClientFactory} */ +const createInstalledClient = (options) => new InstalledSigV4Client(options); +/** @type {TestSigV4ClientFactory} */ +const createVendoredClient = (options) => new SigV4Client(options); +const SIGNER_IMPLEMENTATIONS = [ + { name: "installed package", createClient: createInstalledClient }, + { name: "vendored bundle", createClient: createVendoredClient }, +]; + // Fixed vectors are kept inline so vendored signer changes cannot silently // redefine expected output. const FIXTURES = [ @@ -70,32 +87,46 @@ const FIXTURES = [ }, ]; -test("aws-sigv4 S3 signing matches fixed SigV4 golden vectors", async () => { - for (const fixture of FIXTURES) { - const client = new SigV4Client({ - accessKeyId: ACCESS_KEY_ID, - secretAccessKey: SECRET_ACCESS_KEY, - sessionToken: fixture.sessionToken, - service: "s3", - region: "us-east-1", - cache: new Map(), - retries: 0, - }); - const signed = await client.sign(fixture.url, { - ...fixture.init, - signing: { signingDate: FIXED_DATETIME }, - }); - assert.equal(signed.url, fixture.expectedUrl, fixture.name); - assert.equal(signed.headers.get("authorization"), fixture.expectedAuthorization, fixture.name); - assert.equal(signed.headers.get("x-amz-date"), FIXED_DATETIME, fixture.name); - assert.equal(signed.headers.get("x-amz-content-sha256"), fixture.expectedContentSha256, fixture.name); - assert.equal(signed.headers.get("x-amz-security-token"), fixture.expectedSecurityToken ?? null, fixture.name); +test("aws-sigv4 fetch emits fixed S3 SigV4 golden vectors", async () => { + for (const implementation of SIGNER_IMPLEMENTATIONS) { + for (const fixture of FIXTURES) { + const label = `${implementation.name}: ${fixture.name}`; + /** @type {Request[]} */ + const requests = []; + const client = implementation.createClient({ + accessKeyId: ACCESS_KEY_ID, + secretAccessKey: SECRET_ACCESS_KEY, + sessionToken: fixture.sessionToken, + service: "s3", + region: "us-east-1", + cache: new Map(), + retries: 0, + fetch: async (input) => { + requests.push(/** @type {Request} */ (input)); + return new Response(null, { status: 200 }); + }, + }); + const response = await client.fetch(fixture.url, /** @type {import("@wdl-dev/aws-sigv4").SigV4RequestInit} */ ({ + ...fixture.init, + signing: { signingDate: FIXED_DATETIME }, + })); + assert.equal(response.status, 200, label); + assert.equal(requests.length, 1, label); + const signed = requests[0]; + assert.equal(signed.url, fixture.expectedUrl, label); + assert.equal(signed.method, fixture.init.method ?? "GET", label); + assert.equal(await signed.clone().text(), fixture.init.body ?? "", label); + assert.equal(signed.headers.get("authorization"), fixture.expectedAuthorization, label); + assert.equal(signed.headers.get("x-amz-date"), FIXED_DATETIME, label); + assert.equal(signed.headers.get("x-amz-content-sha256"), fixture.expectedContentSha256, label); + assert.equal(signed.headers.get("x-amz-security-token"), fixture.expectedSecurityToken ?? null, label); + } } }); -/** @param {typeof globalThis.fetch} fetch */ -function testClient(fetch) { - return new SigV4Client({ +/** @param {typeof globalThis.fetch} fetch @param {TestSigV4ClientFactory} [createClient] */ +function testClient(fetch, createClient = createVendoredClient) { + return createClient({ accessKeyId: ACCESS_KEY_ID, secretAccessKey: SECRET_ACCESS_KEY, service: "s3", @@ -122,7 +153,7 @@ test("aws-sigv4 fetch retries idempotent transient responses", async () => { url: request.url, method: request.method, authorization: request.headers.get("authorization"), - body: await request.clone().text(), + body: await request.text(), }); return new Response(null, { status: calls.length === 1 ? 500 : 200 }); }); @@ -145,6 +176,31 @@ test("aws-sigv4 fetch retries idempotent transient responses", async () => { } }); +test("aws-sigv4 retry does not wait for response cancellation", { timeout: 2_000 }, async () => { + for (const implementation of SIGNER_IMPLEMENTATIONS) { + let calls = 0; + let cancellations = 0; + const client = testClient(async () => { + calls += 1; + if (calls === 2) return new Response("ok"); + return new Response(new ReadableStream({ + cancel() { + cancellations += 1; + return new Promise(() => {}); + }, + }), { status: 500 }); + }, implementation.createClient); + + const response = await client.fetch(`${S3_ENDPOINT}/wdl-r2/retry-cancel.txt`, { + signing: { signingDate: FIXED_DATETIME }, + }); + + assert.equal(await response.text(), "ok", implementation.name); + assert.equal(calls, 2, implementation.name); + assert.equal(cancellations, 1, implementation.name); + } +}); + test("aws-sigv4 fetch does not retry POST transient responses", async () => { /** @type {Request[]} */ const calls = []; diff --git a/tests/unit/bounded-body.test.js b/tests/unit/bounded-body.test.js index 5e878b3..155bcc9 100644 --- a/tests/unit/bounded-body.test.js +++ b/tests/unit/bounded-body.test.js @@ -1,6 +1,11 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { BodyTooLargeError, readBoundedBytes, readBoundedText } from "../../shared/bounded-body.js"; +import { + BodyTooLargeError, + readBoundedBytes, + readBoundedStreamBytes, + readBoundedText, +} from "../../shared/bounded-body.js"; test("readBoundedText rejects oversized declared content length", async () => { const request = new Request("https://demo.workers.example", { @@ -39,3 +44,64 @@ test("readBoundedBytes returns an empty body when no request body exists", async const bytes = await readBoundedBytes(request, 1); assert.equal(bytes.byteLength, 0); }); + +test("readBoundedStreamBytes copies a view backed by a larger buffer", async () => { + const backing = new Uint8Array([0, 1, 2, 3]); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(backing.subarray(1, 3)); + controller.close(); + }, + }); + + const bytes = await readBoundedStreamBytes(stream, 2); + + assert.deepEqual([...bytes], [1, 2]); + assert.equal(bytes.buffer.byteLength, 2); +}); + +test("readBoundedStreamBytes supports caller-owned overflow errors", async () => { + let cancelled = false; + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array([1, 2, 3])); + }, + cancel() { + cancelled = true; + }, + }); + + await assert.rejects( + () => readBoundedStreamBytes(stream, 2, () => new TypeError("custom limit")), + /custom limit/ + ); + assert.equal(cancelled, true); +}); + +test("readBoundedStreamBytes rejects without waiting for stream cancellation", async () => { + let cancelled = false; + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array([1, 2, 3])); + }, + cancel() { + cancelled = true; + return new Promise(() => {}); + }, + }); + let timeout; + try { + await assert.rejects( + Promise.race([ + readBoundedStreamBytes(stream, 2, () => new TypeError("custom limit")), + new Promise((_, reject) => { + timeout = setTimeout(() => reject(new Error("body reader waited for stream cancellation")), 1000); + }), + ]), + /custom limit/ + ); + } finally { + clearTimeout(timeout); + } + assert.equal(cancelled, true); +}); diff --git a/tests/unit/control-d1-migrations.test.js b/tests/unit/control-d1-migrations.test.js index 36a691b..c112382 100644 --- a/tests/unit/control-d1-migrations.test.js +++ b/tests/unit/control-d1-migrations.test.js @@ -5,6 +5,7 @@ import { applyModuleReplacements, moduleDataUrl, readRepositoryFile, + repositoryFileUrl, } from "../helpers/load-shared-module.js"; import { readJsonResponse } from "../helpers/response-json.js"; @@ -15,12 +16,19 @@ const D1_MIGRATIONS_TEST_STATE = { runtimeCalls: null, runtimeResult: null, lockToken: null, + logs: [], splitStatements: null, }; /** @type {typeof globalThis & { __d1MigrationTestState?: typeof D1_MIGRATIONS_TEST_STATE }} */ const d1MigrationGlobal = globalThis; d1MigrationGlobal.__d1MigrationTestState = D1_MIGRATIONS_TEST_STATE; +const sharedRandomIdUrl = repositoryFileUrl("shared/random-id.js"); +const sharedRedisLockUrl = moduleDataUrl(applyModuleReplacements( + readRepositoryFile("shared/redis-lock.js"), + [[/from "shared-random-id"/g, `from ${JSON.stringify(sharedRandomIdUrl)}`]] +)); + const controlSharedUrl = controlSharedStubUrl(` export const state = { redis: { @@ -41,7 +49,9 @@ export const state = { return await fn(globalThis.__d1MigrationTestState.session); }, }, - log() {}, + log(level, event, fields) { + /** @type {any} */ (globalThis).__d1MigrationTestState.logs.push({ level, event, fields }); + }, }; `); @@ -108,6 +118,7 @@ const src = applyModuleReplacements(readRepositoryFile("control/d1-migrations.js [/from "control-d1-model";/, `from ${JSON.stringify(modelUrl)};`], [/from "control-d1-runtime-client";/, `from ${JSON.stringify(backendUrl)};`], [/from "control-d1-store";/, `from ${JSON.stringify(storeUrl)};`], + [/from "shared-redis-lock";/, `from ${JSON.stringify(sharedRedisLockUrl)};`], ]); const { @@ -123,6 +134,7 @@ afterEach(() => { D1_MIGRATIONS_TEST_STATE.runtimeCalls = null; D1_MIGRATIONS_TEST_STATE.runtimeResult = null; D1_MIGRATIONS_TEST_STATE.lockToken = null; + D1_MIGRATIONS_TEST_STATE.logs = []; D1_MIGRATIONS_TEST_STATE.splitStatements = null; }); @@ -265,6 +277,7 @@ test("applyMigrations maps lock renewal loss to 409", async () => { test("applyMigrations includes empty progress when reading existing migrations fails", async () => { /** @type {any} */ (globalThis).__d1MigrationTestState.redis = { async set() { return "OK"; }, + async delIfEq() { throw new Error("migration lock release unavailable"); }, }; /** @type {any} */ (globalThis).__d1MigrationTestState.runtimeResult = { ok: false, @@ -290,6 +303,17 @@ test("applyMigrations includes empty progress when reading existing migrations f assert.equal(body.error, "d1_migrations_apply_failed"); assert.deepEqual(body.applied, []); assert.deepEqual(body.skipped, []); + assert.deepEqual(/** @type {any} */ (globalThis).__d1MigrationTestState.logs, [{ + level: "warn", + event: "d1_migration_lock_release_failed", + fields: { + request_id: "rid-migration-read-failure", + namespace: "demo", + database_id: "d1_main", + error_name: "Error", + error_message: "migration lock release unavailable", + }, + }]); } finally { /** @type {any} */ (globalThis).__d1MigrationTestState.redis = null; /** @type {any} */ (globalThis).__d1MigrationTestState.runtimeResult = null; diff --git a/tests/unit/control-d1-model.test.js b/tests/unit/control-d1-model.test.js index d7bba76..3a5ace7 100644 --- a/tests/unit/control-d1-model.test.js +++ b/tests/unit/control-d1-model.test.js @@ -4,6 +4,7 @@ import { importRepositoryModule, repositoryFileUrl } from "../helpers/load-share const SQL_SPLITTER_URL = repositoryFileUrl("shared/sql-splitter.js"); const SHARED_HEX_URL = repositoryFileUrl("shared/hex.js"); +const SHARED_NS_URL = repositoryFileUrl("shared/ns-pattern.js"); const { decodeDatabaseHash, isReadyDatabase, @@ -18,6 +19,7 @@ const { [/export \{ splitSqlStatements \} from "shared-sql-splitter";/, `export { splitSqlStatements } from ${JSON.stringify(SQL_SPLITTER_URL)};`], [/from "shared-hex";/, `from ${JSON.stringify(SHARED_HEX_URL)};`], + [/from "shared-ns-pattern";/, `from ${JSON.stringify(SHARED_NS_URL)};`], ]); test("control D1: database metadata state is explicit", () => { diff --git a/tests/unit/control-d1-runtime-client.test.js b/tests/unit/control-d1-runtime-client.test.js index 0df3d67..976878c 100644 --- a/tests/unit/control-d1-runtime-client.test.js +++ b/tests/unit/control-d1-runtime-client.test.js @@ -29,7 +29,7 @@ const { "shared-d1-query-wire": d1QueryWireDataUrl(), "shared-respond": repositoryFileUrl("shared/respond.js"), "shared-internal-auth": sharedInternalAuthUrl(), - "runtime-owner-endpoint": repositoryFileUrl("runtime/_wdl-owner-endpoint.js"), + "shared-owner-endpoint": repositoryFileUrl("shared/owner-endpoint.js"), })); /** @template {Record} T @param {T} env */ @@ -87,30 +87,28 @@ test("control D1 runtime client query sends request timeout signal", async () => }); }); -test("control D1 runtime client preserves zero owner generation hints", async () => { - const result = await d1RuntimeQuery(withInternalAuthEnv({ - D1_BACKEND: { - async fetch() { - return d1Response( - { success: true, results: [] }, - { - headers: { - "x-wdl-d1-owner-task-id": "task-a", - "x-wdl-d1-owner-endpoint": "10.0.0.9:8787", - "x-wdl-d1-owner-generation": "0", - }, - } - ); +test("control D1 runtime client rejects non-positive and unsafe owner generation hints", async () => { + for (const rawGeneration of ["0", "9007199254740992"]) { + const result = await d1RuntimeQuery(withInternalAuthEnv({ + D1_BACKEND: { + async fetch() { + return d1Response( + { success: true, results: [] }, + { + headers: { + "x-wdl-d1-owner-task-id": "task-a", + "x-wdl-d1-owner-endpoint": "10.0.0.9:8787", + "x-wdl-d1-owner-generation": rawGeneration, + }, + } + ); + }, }, - }, - }), "tenant-a", "db1", "all", [{ sql: "select 1", params: [] }]); + }), "tenant-a", "db1", "all", [{ sql: "select 1", params: [] }]); - assert.equal(result.ok, true); - assert.deepEqual(result.owner, { - taskId: "task-a", - endpoint: "10.0.0.9:8787", - generation: 0, - }); + assert.equal(result.ok, true); + assert.equal(result.owner, null, rawGeneration); + } }); test("control D1 runtime client rejects fractional owner generation hints", async () => { diff --git a/tests/unit/control-d1-store.test.js b/tests/unit/control-d1-store.test.js index a9dc1fe..dbf0c1d 100644 --- a/tests/unit/control-d1-store.test.js +++ b/tests/unit/control-d1-store.test.js @@ -1,7 +1,7 @@ import { afterEach, test } from "node:test"; import assert from "node:assert/strict"; import { controlSharedStubUrl } from "../helpers/control-shared-stub.js"; -import { createFakeRedis } from "../helpers/mocks/fake-redis.js"; +import { createFakeRedis, sharedRedisStubUrl } from "../helpers/mocks/fake-redis.js"; import { applyModuleReplacements, moduleDataUrl, @@ -41,10 +41,7 @@ export function d1DatabaseTombstonesKey(ns) { return "d1:database-tombstones:" + export function formatD1ReferrerBlockers() { return { blockers: [], malformedReferrerCount: 0 }; } `); -const sharedRedisUrl = moduleDataUrl(` -export function decodeBulk(value) { return value; } -export class WatchError extends Error {} -`); +const sharedRedisUrl = sharedRedisStubUrl(); const modelUrl = repositoryModuleDataUrl("control/d1-model.js", [ [ @@ -52,6 +49,7 @@ const modelUrl = repositoryModuleDataUrl("control/d1-model.js", [ `export { splitSqlStatements } from ${JSON.stringify(repositoryFileUrl("shared/sql-splitter.js"))};` ], [/from "shared-hex";/, `from ${JSON.stringify(repositoryFileUrl("shared/hex.js"))};`], + [/from "shared-ns-pattern";/, `from ${JSON.stringify(repositoryFileUrl("shared/ns-pattern.js"))};`], ]); const src = applyModuleReplacements(readRepositoryFile("control/d1-store.js"), [ [/from "control-shared";/, `from ${JSON.stringify(controlSharedUrl)};`], diff --git a/tests/unit/control-delete-handler.test.js b/tests/unit/control-delete-handler.test.js index 92dfd1b..cb4cd9d 100644 --- a/tests/unit/control-delete-handler.test.js +++ b/tests/unit/control-delete-handler.test.js @@ -11,15 +11,23 @@ import { readRepositoryFile, repositoryFileUrl, } from "../helpers/load-shared-module.js"; +import { sharedRedisStubUrl } from "../helpers/mocks/fake-redis.js"; import { assertJsonResponse, readJsonResponse } from "../helpers/response-json.js"; -const { lifecycleIndexesUrl } = await compileControlGraph(); +const { libUrl: productionControlLibUrl, lifecycleIndexesUrl } = await compileControlGraph(); +const sharedVersionUrl = repositoryFileUrl("shared/version.js"); const controlSharedExtraSource = ` -export const ROUTES_CHANNEL = "routes:invalidate"; -export const ROUTES_FLUSH_CHANNEL = "routes:flush"; -export const PATTERNS_CHANNEL = "patterns:invalidate"; -export async function acquireDeleteLock() { return "lock-token"; } +export { + PATTERNS_CHANNEL, + ROUTES_CHANNEL, + ROUTES_FLUSH_CHANNEL, +} from ${JSON.stringify(sharedVersionUrl)}; +export async function acquireDeleteLock(_redis, _ns, _worker, kind) { + /** @type {any} */ (globalThis).__deleteHandlerState.lockKinds.push(kind); + return "lock-token"; +} +export async function renewDeleteLock() { return true; } export async function assertWorkflowDeleteAllowed(args) { /** @type {any} */ (globalThis).__deleteHandlerState.workflowChecks.push(args); if (/** @type {any} */ (globalThis).__deleteHandlerWorkflowBlocker) { @@ -47,7 +55,11 @@ export async function recordS3CleanupIntent(intent) { const versionsControlSharedExtraSource = ` export function hasCompleteBundle() { return true; } -export async function acquireDeleteLock() { return "lock-token"; } +export async function acquireDeleteLock(_redis, _ns, _worker, kind) { + /** @type {any} */ (globalThis).__versionDeleteHandlerState.lockKinds.push(kind); + return "lock-token"; +} +export async function renewDeleteLock() { return true; } export async function assertWorkflowDeleteAllowed(args) { /** @type {any} */ (globalThis).__versionDeleteHandlerState.workflowChecks.push(args); } @@ -62,50 +74,29 @@ export async function recordS3CleanupIntent(intent) { `; const controlLibUrl = moduleDataUrl(` -export function workersIndexKey(ns) { return "workers:" + ns; } -export function workerVersionsKey(ns, name) { return "worker-versions:" + ns + ":" + name; } -export function referrersKey(ns, name, version) { - return "worker-version-referrers:" + ns + ":" + name + ":" + version; -} -export function d1DatabaseReferrersKey(ns, databaseId) { - return "d1:database-referrers:" + ns + ":" + databaseId; -} -export function deleteLockKey(ns, worker) { return "worker-delete-lock:" + ns + ":" + worker; } -export function encodeReferrerMember(ref) { return JSON.stringify(ref); } -export function doObjectRegistryKey(id) { return "do:objects:" + id; } -export function doOwnerScopeScanPatternForStorage(id) { return "do:owner:" + id + ":*"; } -export function doStorageIdKey(ns, worker) { return "worker:do-storage:" + ns + ":" + worker; } -export function workflowDefsKey(ns, worker) { return "wf:defs:" + ns + ":" + worker; } +export { + bundleAssetPrefix, + d1DatabaseReferrersKey, + deleteLockKey, + doObjectRegistryKey, + doOwnerScopeScanPatternForStorage, + doStorageIdKey, + encodeReferrerMember, + parseBundleMeta, + referrersKey, + workerVersionsKey, + workersIndexKey, + workflowDefsKey, +} from ${JSON.stringify(productionControlLibUrl)}; export function extractD1Refs() { return []; } export function extractOutgoingRefs() { return []; } export function formatReferrerBlocker(members) { return { referrers: members }; } `); -const sharedVersionUrl = moduleDataUrl(` -export function routesKey(ns) { return "routes:" + ns; } -export function patternsKey(host) { return "patterns:" + host; } -export function formatVersion(n) { return "v" + n; } -export function parseVersion(tag) { - const match = /^v([1-9][0-9]*)$/.exec(tag); - return match ? Number(match[1]) : null; -} -export function bundleKey(ns, worker, version) { - const n = parseVersion(version); - if (n == null) throw new Error("invalid version tag " + JSON.stringify(version)); - return "worker:" + ns + ":" + worker + ":v:" + n; -} -`); - const sharedSecretKeysUrl = repositoryFileUrl("shared/secret-keys.js"); -const sharedRedisUrl = moduleDataUrl(` -const decoder = new TextDecoder(); -export function decodeBulk(value) { - if (value == null) return null; - return value instanceof Uint8Array ? decoder.decode(value) : String(value); -} -export class WatchError extends Error {} -`); +const sharedRedisUrl = sharedRedisStubUrl(); +const { WatchError: TestWatchError } = await import(sharedRedisUrl); const sharedQueueKeysUrl = moduleDataUrl(` export const QUEUE_CONSUMER_INDEX_KEY = "queue:index:consumers"; @@ -162,39 +153,74 @@ const { handle: handleVersions } = await importControlHandler("control/handlers/ * activeVersion?: string | null, * assetPrefix?: string | null, * hasWorkerSecrets?: boolean, + * hasWorkflowDefs?: boolean, * doStorageId?: string | null, + * doStorageIdDuringExec?: string | null, * doOwnerKeys?: string[], * doOwnerKeysDuringExec?: string[], * queueConsumerKeys?: string[], + * queueConsumerKeysDuringExec?: string[], * queueConsumerWorker?: string | null, * queueConsumerWorkerDuringExec?: string | null, * referrersByVersion?: Record, * retainedVersions?: string[], + * bundleMetaRaw?: string | null, + * lockTokenDuringExec?: string | null, + * replaceLockAfterRead?: boolean, + * siblingVersion?: string | null, + * siblingVersionBeforeFirstWatch?: string | null, + * patternRecords?: Record>, + * patternRecordsDuringExec?: Record>, * }} [opts] */ function resetDeleteHandlerState({ activeVersion = "v1", assetPrefix = "assets/demo/api/v1/", hasWorkerSecrets = false, + hasWorkflowDefs = false, doStorageId = "do_old", + doStorageIdDuringExec = doStorageId, doOwnerKeys = [], doOwnerKeysDuringExec = doOwnerKeys, queueConsumerKeys = ["queue-consumer:demo:jobs"], + queueConsumerKeysDuringExec = queueConsumerKeys, queueConsumerWorker = "api", queueConsumerWorkerDuringExec = queueConsumerWorker, referrersByVersion = {}, retainedVersions = ["v1"], + bundleMetaRaw, + lockTokenDuringExec = "lock-token", + replaceLockAfterRead = false, + siblingVersion = null, + siblingVersionBeforeFirstWatch = null, + patternRecords = {}, + patternRecordsDuringExec = patternRecords, } = {}) { const referrers = /** @type {Record} */ (referrersByVersion); - const meta = JSON.stringify({ - ...(assetPrefix ? { assets: { prefix: assetPrefix } } : {}), - bindings: {}, - routes: [], - }); + const meta = bundleMetaRaw === undefined + ? JSON.stringify({ + ...(assetPrefix ? { assets: { prefix: assetPrefix } } : {}), + bindings: {}, + routes: [], + }) + : bundleMetaRaw; /** @type {unknown[][]} */ const multiCalls = []; /** @type {unknown[][]} */ const commands = []; + /** @type {string[]} */ + let watchedKeys = []; + /** @type {string[][]} */ + const watchBatches = []; + let watchedLockToken = lockTokenDuringExec; + let currentLockToken = lockTokenDuringExec; + let lockReplaced = false; + /** @type {Record} */ + const routeVersions = { + ...(activeVersion ? { api: activeVersion } : {}), + ...(siblingVersion ? { workerB: siblingVersion } : {}), + }; + let siblingChanged = false; /** @param {string} key */ function versionReferrers(key) { const match = /^worker-version-referrers:demo:api:(v[1-9][0-9]*)$/.exec(key); @@ -202,8 +228,34 @@ function resetDeleteHandlerState({ return []; } const session = { - async watch() {}, - async unwatch() {}, + /** @param {string[]} keys */ + async watch(...keys) { + if ( + siblingVersionBeforeFirstWatch && + !siblingChanged && + keys.includes("routes:demo") + ) { + routeVersions.workerB = siblingVersionBeforeFirstWatch; + siblingChanged = true; + } + watchedKeys = keys; + watchBatches.push(keys); + watchedLockToken = currentLockToken; + }, + async unwatch() { watchedKeys = []; }, + /** @param {string} key */ + async get(key) { + if (key === "worker-delete-lock:demo:api") { + const token = currentLockToken; + if (replaceLockAfterRead && !lockReplaced) { + currentLockToken = "replacement-token"; + lockReplaced = true; + } + return token; + } + if (key === "worker:do-storage:demo:api") return doStorageIdDuringExec; + return null; + }, /** @param {string} key */ async zRange(key) { if (key === "worker-versions:demo:api") return retainedVersions; @@ -211,7 +263,8 @@ function resetDeleteHandlerState({ }, /** @param {string} key @param {string} field */ async hGet(key, field) { - if (key === "routes:demo" && field === "api") return activeVersion; + if (key === "routes:demo") return routeVersions[field] || null; + if (key.startsWith("patterns:")) return patternRecordsDuringExec[key]?.[field] || null; if (queueConsumerKeys.includes(key) && field === "worker") { return queueConsumerWorkerDuringExec; } @@ -226,26 +279,32 @@ function resetDeleteHandlerState({ }, /** @param {string} key */ async exists(key) { - return key === "secrets:demo:api" && hasWorkerSecrets ? 1 : 0; + if (key === "secrets:demo:api") return hasWorkerSecrets ? 1 : 0; + if (key === "wf:defs:demo:api") return hasWorkflowDefs ? 1 : 0; + return 0; }, /** @param {string} key */ async sMembers(key) { return versionReferrers(key); }, /** @param {string[]} keys */ async hGetAllMany(keys) { commands.push(["SESSION_HGETALLMANY", keys]); - return keys.map(() => ({})); + return keys.map((key) => patternRecordsDuringExec[key] || {}); }, /** @param {string} _cursor @param {string} pattern */ async scan(_cursor, pattern) { - if (pattern === "do:owner:do_old:*") return ["0", doOwnerKeysDuringExec]; + if (pattern === "do:owner:scope:do_old%3A*") return ["0", doOwnerKeysDuringExec]; if (pattern === "queue-consumer:demo:*" && queueConsumerWorkerDuringExec) { - return ["0", queueConsumerKeys]; + return ["0", queueConsumerKeysDuringExec]; } return ["0", []]; }, /** @param {unknown[]} args */ async del(...args) { multiCalls.push(["DEL", ...args]); }, - async hGetAll() { return {}; }, + /** @param {string} key */ + async hGetAll(key) { + if (key === "routes:demo") return { ...routeVersions }; + return {}; + }, multi() { return { /** @param {unknown[]} args */ @@ -258,7 +317,15 @@ function resetDeleteHandlerState({ zRem(...args) { multiCalls.push(["ZREM", ...args]); return this; }, /** @param {unknown[]} args */ publish(...args) { multiCalls.push(["PUBLISH", ...args]); return this; }, - async exec() { multiCalls.push(["EXEC"]); }, + async exec() { + if ( + watchedKeys.includes("worker-delete-lock:demo:api") && + currentLockToken !== watchedLockToken + ) { + throw new TestWatchError(); + } + multiCalls.push(["EXEC"]); + }, }; }, }; @@ -270,7 +337,7 @@ function resetDeleteHandlerState({ }, /** @param {string} _cursor @param {string} pattern */ async scan(_cursor, pattern) { - if (pattern === "do:owner:do_old:*") return ["0", doOwnerKeys]; + if (pattern === "do:owner:scope:do_old%3A*") return ["0", doOwnerKeys]; if (pattern === "queue-consumer:demo:*" && queueConsumerWorker) { return ["0", queueConsumerKeys]; } @@ -283,6 +350,7 @@ function resetDeleteHandlerState({ }, /** @param {string} key @param {string} field */ async hGet(key, field) { + if (key === "routes:demo") return routeVersions[field] || null; if (/^worker:demo:api:v:\d+$/.test(key) && field === "__meta__") return meta; if (queueConsumerKeys.includes(key) && field === "worker") return queueConsumerWorker; return null; @@ -298,7 +366,7 @@ function resetDeleteHandlerState({ async sMembers(key) { return versionReferrers(key); }, /** @param {string} key */ async hGetAll(key) { - if (key === "routes:demo" && activeVersion) return { api: activeVersion }; + if (key === "routes:demo") return { ...routeVersions }; return {}; }, /** @param {string[]} keys */ @@ -306,12 +374,14 @@ function resetDeleteHandlerState({ commands.push(["CLIENT_HGETALLMANY", keys]); return keys.map((key) => { if (key === "routes:demo" && activeVersion) return { api: activeVersion }; - return {}; + return patternRecords[key] || {}; }); }, /** @param {string} key */ async exists(key) { - return key === "secrets:demo:api" && hasWorkerSecrets ? 1 : 0; + if (key === "secrets:demo:api") return hasWorkerSecrets ? 1 : 0; + if (key === "wf:defs:demo:api") return hasWorkflowDefs ? 1 : 0; + return 0; }, /** @param {unknown[]} args */ async del(...args) { multiCalls.push(["DEL", ...args]); }, @@ -323,39 +393,63 @@ function resetDeleteHandlerState({ cleanupIntents: [], doAlarmCleanups: [], logs: [], + lockKinds: [], multiCalls, releaseCalls: 0, redis, metrics: { increment() {}, observe() {} }, service: "control", workflowChecks: [], + watchBatches, + get watchedKeys() { return watchedKeys; }, }); } -/** @param {{ assetPrefix?: string | null }} [opts] */ -function resetVersionDeleteHandlerState({ assetPrefix = "assets/demo/api/v1/" } = {}) { - const meta = JSON.stringify({ - ...(assetPrefix ? { assets: { prefix: assetPrefix } } : {}), - bindings: {}, - }); +/** @param {{ assetPrefix?: string | null, bundleMetaRaw?: string | null, retainedVersions?: string[], siblingMetaRaw?: string | null, lockTokenDuringExec?: string | null, hasWorkflowDefs?: boolean }} [opts] */ +function resetVersionDeleteHandlerState({ + assetPrefix = "assets/demo/api/v1/", + bundleMetaRaw, + retainedVersions = ["v1"], + siblingMetaRaw = null, + lockTokenDuringExec = "lock-token", + hasWorkflowDefs = false, +} = {}) { + const meta = bundleMetaRaw === undefined + ? JSON.stringify({ + ...(assetPrefix ? { assets: { prefix: assetPrefix } } : {}), + bindings: {}, + }) + : bundleMetaRaw; /** @type {unknown[][]} */ const multiCalls = []; + /** @type {string[][]} */ + const watchBatches = []; const session = { - async watch() {}, + /** @param {string[]} keys */ + async watch(...keys) { watchBatches.push(keys); }, async unwatch() {}, + /** @param {string} key */ + async get(key) { + if (key === "worker-delete-lock:demo:api") return lockTokenDuringExec; + return null; + }, /** @param {string} key @param {string} field */ async hGet(key, field) { if (key === "routes:demo" && field === "api") return null; if (key === "worker:demo:api:v:1" && field === "__meta__") return meta; + if (key === "worker:demo:api:v:2" && field === "__meta__") return siblingMetaRaw; return null; }, /** @param {string} key */ async zRange(key) { - if (key === "worker-versions:demo:api") return ["v1"]; + if (key === "worker-versions:demo:api") return retainedVersions; return []; }, async sMembers() { return []; }, - async exists() { return 0; }, + /** @param {string} key */ + async exists(key) { + return key === "wf:defs:demo:api" && hasWorkflowDefs ? 1 : 0; + }, multi() { return { /** @param {unknown[]} args */ @@ -375,11 +469,13 @@ function resetVersionDeleteHandlerState({ assetPrefix = "assets/demo/api/v1/" } return installControlHandlerState("__versionDeleteHandlerState", { cleanupIntents: [], logs: [], + lockKinds: [], multiCalls, releaseCalls: 0, redis, metrics: { increment() {}, observe() {} }, service: "control", + watchBatches, workflowChecks: [], }); } @@ -457,8 +553,13 @@ test("worker delete reports cleanup_queue_failed when data-plane cleanup enqueue const body = await readJsonResponse(response, 200); assert.equal(body.deleted, true); - assert.deepEqual(testState.workflowChecks, [{ ns: "demo", worker: "api", allowCleanup: true }]); - assert.deepEqual(testState.doAlarmCleanups, [{ ns: "demo", worker: "api", doStorageId: "do_old" }]); + assert.deepEqual(testState.lockKinds, ["whole"]); + assert.deepEqual(testState.workflowChecks, [{ + ns: "demo", worker: "api", allowCleanup: true, requestId: "rid-delete", + }]); + assert.deepEqual(testState.doAlarmCleanups, [{ + ns: "demo", worker: "api", doStorageId: "do_old", requestId: "rid-delete", + }]); assert.equal(body.assets.queueHint, "failed"); assert.deepEqual(body.assets.warnings, [{ code: "cleanup_queue_failed", @@ -507,6 +608,110 @@ test("worker delete reports cleanup_queue_failed when data-plane cleanup enqueue )); }); +test("worker delete classifies non-object retained bundle metadata as corrupt", async () => { + const testState = resetDeleteHandlerState({ bundleMetaRaw: "[]" }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-delete-corrupt-meta", + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_meta"); + assert.equal(body.stage, undefined); + assert.equal(body.detail, undefined); + const rejection = /** @type {any} */ (testState.logs.find((/** @type {any} */ entry) => + entry.event === "worker_delete_rejected" + )); + assert.ok(rejection); + assert.equal(rejection.level, "error"); + assert.equal(rejection.fields.metadata_version, "v1"); + assert.equal(rejection.fields.stage, "retained_meta_parse"); + assert.equal(rejection.fields.error_detail, "__meta__ must be a JSON object"); + assert.equal(testState.multiCalls.some((/** @type {any} */ call) => call[0] === "EXEC"), false); + assert.equal(testState.releaseCalls, 1); +}); + +test("worker delete fails closed when indexed active bundle metadata is missing", async () => { + const testState = resetDeleteHandlerState({ + retainedVersions: [], + bundleMetaRaw: null, + }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-delete-missing-active-meta", + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_meta"); + assert.equal(body.stage, undefined); + assert.equal(testState.multiCalls.some((/** @type {any} */ call) => call[0] === "EXEC"), false); + assert.equal(testState.releaseCalls, 1); +}); + +test("worker delete dry-run logs redacted metadata diagnostics", async () => { + const testState = resetDeleteHandlerState({ bundleMetaRaw: "[]" }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete?dry_run=1", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete?dry_run=1"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-delete-dry-run-corrupt-meta", + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_meta"); + assert.equal(body.stage, undefined); + assert.equal(body.detail, undefined); + const rejection = /** @type {any} */ (testState.logs.find((/** @type {any} */ entry) => + entry.event === "worker_dry_run_rejected" + )); + assert.ok(rejection); + assert.equal(rejection.level, "error"); + assert.equal(rejection.fields.metadata_version, "v1"); + assert.equal(rejection.fields.stage, "retained_meta_parse"); + assert.equal(rejection.fields.error_detail, "__meta__ must be a JSON object"); +}); + +for (const { label, bundleMetaRaw } of [ + { label: "missing", bundleMetaRaw: null }, + { label: "empty", bundleMetaRaw: "" }, +]) { + test(`worker delete fails closed when retained bundle metadata is ${label}`, async () => { + const testState = resetDeleteHandlerState({ + activeVersion: null, + bundleMetaRaw, + }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: `rid-delete-${label}-retained-meta`, + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_meta"); + assert.equal(body.stage, undefined); + assert.equal(body.detail, undefined); + assert.equal(testState.multiCalls.some((/** @type {any} */ call) => call[0] === "EXEC"), false); + assert.equal(testState.releaseCalls, 1); + }); +} + test("worker delete collects retained version metadata without serial round trips", async () => { const testState = resetDeleteHandlerState({ activeVersion: null, @@ -539,6 +744,7 @@ test("worker delete collects retained version metadata without serial round trip const body = await readJsonResponse(response, 200); assert.equal(body.deleted, true); + assert.deepEqual(testState.lockKinds, ["whole"]); assert.deepEqual(body.versionsDeleted, ["v1", "v2", "v3"]); assert.ok( maxActiveBundleMetaReads > 1, @@ -550,7 +756,7 @@ test("worker delete retries instead of committing when DO owner keys drift after const testState = resetDeleteHandlerState({ assetPrefix: null, doOwnerKeys: [], - doOwnerKeysDuringExec: ["do:owner:do_old:Room:shard0"], + doOwnerKeysDuringExec: ["do:owner:scope:do_old%3ARoom%3Ashard0"], }); const response = await handle({ @@ -598,6 +804,268 @@ test("worker delete retries instead of committing when queue consumer keys drift assert.deepEqual(testState.doAlarmCleanups, []); }); +test("worker delete deduplicates Redis SCAN results before drift checks and counts", async () => { + const ownerKey = "do:owner:scope:do_old%3ARoom%3Ashard0"; + const queueKey = "queue-consumer:demo:jobs"; + const testState = resetDeleteHandlerState({ + assetPrefix: null, + doOwnerKeys: [ownerKey, ownerKey], + doOwnerKeysDuringExec: [ownerKey], + queueConsumerKeys: [queueKey, queueKey], + queueConsumerKeysDuringExec: [queueKey], + }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-delete-scan-duplicates", + }); + + const body = await readJsonResponse(response, 200); + assert.equal(body.deleted, true); + assert.equal(body.queueConsumersRemoved, 1); + assert.deepEqual( + testState.commands.find((/** @type {unknown[]} */ call) => call[0] === "CLIENT_HGETMANY"), + ["CLIENT_HGETMANY", [[queueKey, "worker"]]], + ); + assert.equal( + (testState.watchBatches.at(-1) || []).filter((key) => key === ownerKey).length, + 1, + ); +}); + +test("worker delete uses the under-WATCH sibling route snapshot", async () => { + const testState = resetDeleteHandlerState({ + assetPrefix: null, + siblingVersionBeforeFirstWatch: "v2", + }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-delete-sibling-promote-race", + }); + + const body = await readJsonResponse(response, 200); + assert.equal(body.deleted, true); + assert.equal(body.namespaceStillActive, true); + assert.equal(testState.watchBatches.length, 1); + assert.equal(testState.multiCalls.some((/** @type {any} */ call) => + call[0] === "SREM" && call[1] === "namespaces" && call[2] === "demo" + ), false); + assert.equal(testState.multiCalls.some((/** @type {any} */ call) => + call[0] === "PUBLISH" && call[1] === "routes:flush" + ), false); + assert.ok(testState.multiCalls.some((/** @type {any} */ call) => + call[0] === "PUBLISH" && call[1] === "routes:invalidate" && call[2] === "demo" + )); +}); + +test("worker delete ignores an unrelated sibling version change before WATCH", async () => { + const testState = resetDeleteHandlerState({ + assetPrefix: null, + siblingVersion: "v2", + siblingVersionBeforeFirstWatch: "v3", + }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-delete-sibling-version-race", + }); + + const body = await readJsonResponse(response, 200); + assert.equal(body.deleted, true); + assert.equal(body.namespaceStillActive, true); + assert.equal(testState.watchBatches.length, 1); +}); + +test("worker delete recomputes sibling host ownership under WATCH", async () => { + const host = "app.example"; + const testState = resetDeleteHandlerState({ + assetPrefix: null, + bundleMetaRaw: JSON.stringify({ + bindings: {}, + routes: [{ host, slot: "target-slot" }], + }), + siblingVersionBeforeFirstWatch: "v2", + patternRecordsDuringExec: { + [`patterns:${host}`]: { + "sibling-slot": "v2\tdemo\tworkerB\tv2\texact\t/other", + }, + }, + }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-delete-sibling-host-race", + }); + + const body = await readJsonResponse(response, 200); + assert.equal(body.namespaceStillActive, true); + assert.equal(testState.watchBatches.length, 1); + assert.equal(testState.multiCalls.some((/** @type {any} */ call) => + call[0] === "SREM" && call[1] === "ns-hosts:demo" && call.includes(host) + ), false); +}); + +test("worker delete fails closed on malformed held pattern projections", async () => { + const host = "app.example"; + const slot = "target-slot"; + const testState = resetDeleteHandlerState({ + assetPrefix: null, + bundleMetaRaw: JSON.stringify({ + bindings: {}, + routes: [{ host, slot }], + }), + patternRecordsDuringExec: { + [`patterns:${host}`]: { [slot]: "not-a-projection" }, + }, + }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-delete-corrupt-pattern", + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_pattern_projection"); + assert.equal(testState.multiCalls.some((call) => call[0] === "EXEC"), false); +}); + +test("worker delete cannot commit after its delete lock token is replaced", async () => { + const testState = resetDeleteHandlerState({ + assetPrefix: null, + lockTokenDuringExec: "replacement-token", + }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-delete-lock-replaced", + }); + + const body = await readJsonResponse(response, 409); + assert.equal(body.error, "deleting"); + assert.equal(testState.multiCalls.some((/** @type {any} */ call) => call[0] === "EXEC"), false); + assert.deepEqual(testState.doAlarmCleanups, []); +}); + +test("worker delete WATCH rejects a lock replacement after the token read", async () => { + const testState = resetDeleteHandlerState({ + assetPrefix: null, + replaceLockAfterRead: true, + }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-delete-lock-replaced-after-read", + }); + + const body = await readJsonResponse(response, 409); + assert.equal(body.error, "deleting"); + assert.ok(testState.watchBatches.some((/** @type {string[]} */ keys) => + keys.includes("worker-delete-lock:demo:api") + )); + assert.equal(testState.multiCalls.some((/** @type {any} */ call) => call[0] === "EXEC"), false); +}); + +test("worker delete rejects an active version missing from the retained index", async () => { + const testState = resetDeleteHandlerState({ + assetPrefix: null, + retainedVersions: [], + }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-delete-active-index-drift", + }); + + const body = await readJsonResponse(response, 409); + assert.equal(body.error, "projection_drift"); + assert.equal(body.active_version, "v1"); + assert.equal(testState.multiCalls.some((/** @type {any} */ call) => call[0] === "EXEC"), false); +}); + +test("worker delete dry-run rejects an active version missing from the retained index", async () => { + const testState = resetDeleteHandlerState({ + assetPrefix: null, + retainedVersions: [], + }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete?dry_run=1", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete?dry_run=1"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-dry-run-active-index-drift", + }); + + const body = await readJsonResponse(response, 409); + assert.equal(body.error, "projection_drift"); + assert.equal(body.active_version, "v1"); + assert.equal(body.dryRun, true); + assert.equal(testState.releaseCalls, 0); +}); + +test("worker delete dry-run retries an active/index snapshot split by a secret bump", async () => { + const testState = resetDeleteHandlerState({ + activeVersion: "v2", + assetPrefix: null, + retainedVersions: ["v1", "v2"], + }); + let versionReads = 0; + testState.redis.zRange = async (/** @type {string} */ key) => { + assert.equal(key, "worker-versions:demo:api"); + versionReads += 1; + return versionReads === 1 ? ["v1"] : ["v1", "v2"]; + }; + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete?dry_run=1", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete?dry_run=1"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-dry-run-secret-bump-race", + }); + + const body = await readJsonResponse(response, 200); + assert.equal(body.dryRun, true); + assert.deepEqual(body.versionsDeleted, ["v1", "v2"]); + assert.equal(versionReads, 3); + assert.equal(testState.releaseCalls, 0); +}); + test("worker delete dry-run includes workflow lifecycle blockers", async () => { const testState = resetDeleteHandlerState(); /** @type {any} */ (globalThis).__deleteHandlerWorkflowBlocker = { @@ -622,7 +1090,9 @@ test("worker delete dry-run includes workflow lifecycle blockers", async () => { delete /** @type {any} */ (globalThis).__deleteHandlerWorkflowBlocker; const body = await readJsonResponse(response, 200); assert.equal(body.deleted, false); - assert.deepEqual(testState.workflowChecks, [{ ns: "demo", worker: "api" }]); + assert.deepEqual(testState.workflowChecks, [{ + ns: "demo", worker: "api", requestId: "rid-dry-run-workflows", + }]); assert.deepEqual(body.workflowBlocker, { error: "workflow_instances_active", message: "demo/api has active workflow instances", @@ -632,6 +1102,37 @@ test("worker delete dry-run includes workflow lifecycle blockers", async () => { assert.equal(testState.releaseCalls, 0); }); +test("worker delete dry-run retries when a retained version is deleted during collection", async () => { + const testState = resetDeleteHandlerState({ + activeVersion: null, + bundleMetaRaw: null, + doStorageId: null, + queueConsumerWorker: null, + }); + let versionReads = 0; + testState.redis.zRange = async (/** @type {string} */ key) => { + assert.equal(key, "worker-versions:demo:api"); + versionReads += 1; + return versionReads === 1 ? ["v1"] : []; + }; + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete?dry_run=1", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete?dry_run=1"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-dry-run-version-delete-race", + }); + + const body = await readJsonResponse(response, 200); + assert.equal(body.deleted, false); + assert.equal(body.noop, true); + assert.deepEqual(body.versionsDeleted, []); + assert.equal(versionReads, 3); + assert.equal(testState.releaseCalls, 0); +}); + test("worker delete reports workflow and version referrer blockers together", async () => { const referrer = JSON.stringify({ callerNs: "demo", @@ -674,6 +1175,7 @@ test("worker delete reports workflow and version referrer blockers together", as ns: "demo", worker: "api", allowCleanup: true, + requestId: "rid-delete-both-blockers", }]); assert.deepEqual(testState.doAlarmCleanups, []); assert.equal(testState.releaseCalls, 1); @@ -713,6 +1215,7 @@ test("worker delete keeps DO alarm jobs when version referrers block deletion", ns: "demo", worker: "api", allowCleanup: true, + requestId: "rid-delete-version-blocker", }]); assert.deepEqual(testState.doAlarmCleanups, []); assert.equal(testState.multiCalls.length, 0); @@ -734,7 +1237,12 @@ test("worker delete logs post-commit DO alarm cleanup failure without rolling ba const body = await readJsonResponse(response, 200); assert.equal(body.deleted, true); - assert.deepEqual(testState.doAlarmCleanups, [{ ns: "demo", worker: "api", doStorageId: "do_old" }]); + assert.deepEqual(testState.doAlarmCleanups, [{ + ns: "demo", + worker: "api", + doStorageId: "do_old", + requestId: "rid-delete-alarm-cleanup-failed", + }]); assert.ok(testState.multiCalls.some((/** @type {any} */ call) => call[0] === "EXEC")); assert.ok(testState.logs.some((/** @type {any} */ entry) => entry.level === "warn" && @@ -781,6 +1289,7 @@ test("worker delete keeps workflow blocker even when worker lifecycle is already ns: "demo", worker: "api", allowCleanup: true, + requestId: "rid-delete-workflow-no-lifecycle", }]); assert.equal(testState.releaseCalls, 1); assert.deepEqual(testState.cleanupIntents, []); @@ -809,8 +1318,40 @@ test("worker delete retry compensates DO alarm cleanup when stale storage pointe const body = await readJsonResponse(response, 200); assert.equal(body.deleted, false); - assert.deepEqual(testState.doAlarmCleanups, [{ ns: "demo", worker: "api", doStorageId: "do_old" }]); - assert.deepEqual(testState.multiCalls, [["DEL", "worker:do-storage:demo:api"]]); + assert.deepEqual(testState.doAlarmCleanups, [{ + ns: "demo", + worker: "api", + doStorageId: "do_old", + requestId: "rid-delete-noop-alarm-cleanup", + }]); + assert.deepEqual(testState.multiCalls, [ + ["DEL", "worker:do-storage:demo:api"], + ["EXEC"], + ]); +}); + +test("worker delete noop cannot clean residual state after its lock is replaced", async () => { + const testState = resetDeleteHandlerState({ + activeVersion: null, + assetPrefix: null, + queueConsumerWorker: null, + retainedVersions: [], + lockTokenDuringExec: "replacement-token", + }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-delete-noop-lock-replaced", + }); + + const body = await readJsonResponse(response, 409); + assert.equal(body.error, "deleting"); + assert.deepEqual(testState.multiCalls, []); + assert.deepEqual(testState.doAlarmCleanups, []); }); test("worker delete noop skips DO alarm cleanup when old storage id is absent", async () => { @@ -837,6 +1378,59 @@ test("worker delete noop skips DO alarm cleanup when old storage id is absent", assert.equal(testState.multiCalls.length, 0); }); +test("worker delete removes orphaned workflow definitions without active versions", async () => { + const testState = resetDeleteHandlerState({ + activeVersion: null, + assetPrefix: null, + doStorageId: null, + queueConsumerWorker: null, + retainedVersions: [], + hasWorkflowDefs: true, + }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-delete-orphan-workflow-defs", + }); + + const body = await readJsonResponse(response, 200); + assert.equal(body.deleted, true); + assert.ok(testState.watchBatches.some((keys) => keys.includes("wf:defs:demo:api"))); + assert.ok(testState.multiCalls.some((call) => + call[0] === "DEL" && call.includes("wf:defs:demo:api") + )); +}); + +test("worker delete dry-run reports orphaned workflow definitions", async () => { + const testState = resetDeleteHandlerState({ + activeVersion: null, + assetPrefix: null, + doStorageId: null, + queueConsumerWorker: null, + retainedVersions: [], + hasWorkflowDefs: true, + }); + + const response = await handle({ + request: new Request("http://control/ns/demo/worker/api/delete?dry_run=1", { method: "POST" }), + url: new URL("http://control/ns/demo/worker/api/delete?dry_run=1"), + ns: "demo", + name: "api", + principal: { kind: "ops" }, + requestId: "rid-delete-dry-run-orphan-workflow-defs", + }); + + const body = await readJsonResponse(response, 200); + assert.equal(body.deleted, true); + assert.equal(body.hasWorkflowDefs, true); + assert.deepEqual(body.versionsDeleted, []); + assert.equal(testState.multiCalls.length, 0); +}); + test("worker delete reports queueHint none when no content cleanup is needed", async () => { const testState = resetDeleteHandlerState({ assetPrefix: null }); @@ -851,8 +1445,12 @@ test("worker delete reports queueHint none when no content cleanup is needed", a const body = await readJsonResponse(response, 200); assert.equal(body.deleted, true); - assert.deepEqual(testState.workflowChecks, [{ ns: "demo", worker: "api", allowCleanup: true }]); - assert.deepEqual(testState.doAlarmCleanups, [{ ns: "demo", worker: "api", doStorageId: "do_old" }]); + assert.deepEqual(testState.workflowChecks, [{ + ns: "demo", worker: "api", allowCleanup: true, requestId: "rid-delete-no-cleanup", + }]); + assert.deepEqual(testState.doAlarmCleanups, [{ + ns: "demo", worker: "api", doStorageId: "do_old", requestId: "rid-delete-no-cleanup", + }]); assert.equal(body.assets.queueHint, "none"); assert.deepEqual(body.assets.warnings, []); assert.deepEqual(testState.cleanupIntents, []); @@ -878,11 +1476,13 @@ test("version delete reports cleanup_queue_failed when data-plane cleanup enqueu const body = await readJsonResponse(response, 200); assert.equal(body.deleted, true); + assert.deepEqual(testState.lockKinds, ["version"]); assert.deepEqual(testState.workflowChecks, [{ ns: "demo", worker: "api", version: "v1", allowCleanup: true, + requestId: "rid-version-delete", }]); assert.equal(body.assets.queueHint, "failed"); assert.deepEqual(body.assets.warnings, [{ @@ -908,6 +1508,135 @@ test("version delete reports cleanup_queue_failed when data-plane cleanup enqueu )); }); +test("version delete cannot commit after its delete lock token is replaced", async () => { + const testState = resetVersionDeleteHandlerState({ + lockTokenDuringExec: "replacement-token", + }); + + const response = await handleVersions({ + method: "DELETE", + ns: "demo", + name: "api", + subPath: ["v1"], + principal: { kind: "ops" }, + requestId: "rid-version-delete-lock-replaced", + }); + + const body = await readJsonResponse(response, 409); + assert.equal(body.error, "deleting"); + assert.equal(testState.multiCalls.some((/** @type {any} */ call) => call[0] === "EXEC"), false); + assert.deepEqual(testState.cleanupIntents, []); +}); + +test("version delete keeps a definitions-only worker discoverable", async () => { + const testState = resetVersionDeleteHandlerState({ + assetPrefix: null, + hasWorkflowDefs: true, + }); + + const response = await handleVersions({ + method: "DELETE", + ns: "demo", + name: "api", + subPath: ["v1"], + principal: { kind: "ops" }, + requestId: "rid-version-delete-workflow-defs", + }); + + await readJsonResponse(response, 200); + assert.ok(testState.watchBatches.some((keys) => keys.includes("wf:defs:demo:api"))); + assert.equal(testState.multiCalls.some((call) => + call[0] === "SREM" && call[1] === "workers:demo" && call[2] === "api" + ), false); +}); + +test("version delete classifies non-object bundle metadata as corrupt", async () => { + const testState = resetVersionDeleteHandlerState({ bundleMetaRaw: "null" }); + + const response = await handleVersions({ + method: "DELETE", + ns: "demo", + name: "api", + subPath: ["v1"], + principal: { kind: "ops" }, + requestId: "rid-version-delete-corrupt-meta", + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_meta"); + assert.equal(body.namespace, "demo"); + assert.equal(body.name, "api"); + assert.equal(body.version, "v1"); + const rejection = /** @type {any} */ (testState.logs.find((/** @type {any} */ entry) => + entry.event === "version_delete_rejected" + )); + assert.ok(rejection); + assert.equal(rejection.level, "error"); + assert.equal(rejection.fields.metadata_version, "v1"); + assert.equal(rejection.fields.stage, "target_meta_parse"); + assert.equal(rejection.fields.error_detail, "__meta__ must be a JSON object"); + assert.deepEqual(testState.cleanupIntents, []); + assert.equal(testState.releaseCalls, 1); +}); + +test("version delete classifies indexed target metadata absence as corrupt", async () => { + const testState = resetVersionDeleteHandlerState({ bundleMetaRaw: null }); + + const response = await handleVersions({ + method: "DELETE", + ns: "demo", + name: "api", + subPath: ["v1"], + principal: { kind: "ops" }, + requestId: "rid-version-delete-missing-target-meta", + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_meta"); + assert.equal(body.version, "v1"); + assert.deepEqual(testState.cleanupIntents, []); + assert.equal(testState.multiCalls.some((/** @type {any} */ call) => call[0] === "EXEC"), false); + assert.equal(testState.releaseCalls, 1); +}); + +for (const [label, siblingMetaRaw] of [ + ["missing", null], + ["empty", ""], +]) { + test(`version delete fails closed when indexed sibling metadata is ${label}`, async () => { + const testState = resetVersionDeleteHandlerState({ + retainedVersions: ["v1", "v2"], + siblingMetaRaw, + }); + + const response = await handleVersions({ + method: "DELETE", + ns: "demo", + name: "api", + subPath: ["v1"], + principal: { kind: "ops" }, + requestId: `rid-version-delete-${label}-sibling-meta`, + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_meta"); + assert.equal(body.version, "v2"); + assert.equal(body.stage, undefined); + assert.equal(body.detail, undefined); + const rejection = /** @type {any} */ (testState.logs.find((/** @type {any} */ entry) => + entry.event === "version_delete_rejected" + )); + assert.ok(rejection); + assert.equal(rejection.level, "error"); + assert.equal(rejection.fields.version, "v1"); + assert.equal(rejection.fields.metadata_version, "v2"); + assert.equal(rejection.fields.stage, "sibling_meta_parse"); + assert.equal(typeof rejection.fields.error_detail, "string"); + assert.deepEqual(testState.cleanupIntents, []); + assert.equal(testState.releaseCalls, 1); + }); +} + test("version delete reports queueHint none when no content cleanup is needed", async () => { const testState = resetVersionDeleteHandlerState({ assetPrefix: null }); @@ -927,6 +1656,7 @@ test("version delete reports queueHint none when no content cleanup is needed", worker: "api", version: "v1", allowCleanup: true, + requestId: "rid-version-delete-no-cleanup", }]); assert.equal(body.assets.queueHint, "none"); assert.deepEqual(body.assets.warnings, []); diff --git a/tests/unit/control-deploy-watch.test.js b/tests/unit/control-deploy-watch.test.js index 8add919..19c5e0b 100644 --- a/tests/unit/control-deploy-watch.test.js +++ b/tests/unit/control-deploy-watch.test.js @@ -11,11 +11,15 @@ import { readRepositoryModuleSource, repositoryFileUrl, } from "../helpers/load-shared-module.js"; -import { createFakeRedisSession } from "../helpers/mocks/fake-redis.js"; +import { createFakeRedisSession, sharedRedisStubUrl } from "../helpers/mocks/fake-redis.js"; import { readJsonResponse } from "../helpers/response-json.js"; import { realRuntimeInjectionSourcesUrl } from "../helpers/runtime-injection-sources.js"; -const { libUrl: controlLibUrl, lifecycleIndexesUrl } = await compileControlGraph(); +const { + libUrl: controlLibUrl, + lifecycleIndexesUrl, + sharedAuthRolesUrl, +} = await compileControlGraph(); /** @type {any} */ const CONTROL_DEPLOY_TEST_STATE = { @@ -38,6 +42,7 @@ const CONTROL_DEPLOY_TEST_STATE = { watchedKeys: null, envBudgetError: false, envBudgetCalls: [], + secretEnvelopeError: null, redis: null, logs: [], metrics: { increment() {}, observe() {} }, @@ -66,6 +71,7 @@ function resetControlDeployTestState() { CONTROL_DEPLOY_TEST_STATE.watchedKeys = null; CONTROL_DEPLOY_TEST_STATE.envBudgetError = false; CONTROL_DEPLOY_TEST_STATE.envBudgetCalls = []; + CONTROL_DEPLOY_TEST_STATE.secretEnvelopeError = null; CONTROL_DEPLOY_TEST_STATE.logs = []; CONTROL_DEPLOY_TEST_STATE.metrics = { increment() {}, observe() {} }; CONTROL_DEPLOY_TEST_STATE.service = "control"; @@ -193,36 +199,11 @@ export function parseQueueConsumers() { } `); -const sharedVersionUrl = moduleDataUrl(` -export function routesKey(ns) { return "routes:" + ns; } -export function formatVersion(n) { return "v" + n; } -export function parseVersion(tag) { - const match = /^v([1-9][0-9]*)$/.exec(tag); - return match ? Number(match[1]) : null; -} -export function bundleKey(ns, worker, version) { - const n = parseVersion(version); - if (n == null) throw new Error("invalid version tag " + JSON.stringify(version)); - return "worker:" + ns + ":" + worker + ":v:" + n; -} -`); +const sharedVersionUrl = repositoryFileUrl("shared/version.js"); -const sharedRedisUrl = moduleDataUrl(` -export class WatchError extends Error {} -`); +const sharedRedisUrl = sharedRedisStubUrl(); -const sharedNsUrl = moduleDataUrl(` -export function isReservedNs(ns) { return typeof ns === "string" && ns.startsWith("__"); } -export function isValidRouteNs(ns) { - return typeof ns === "string" && - (/^[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?$/.test(ns) || ns === "__system__"); -} -export const ROUTES_ALLOWED_RESERVED_NS = new Set(["__system__"]); -`); - -const sharedAuthRolesUrl = moduleDataUrl(` -export const PLATFORM_TIER_RESERVED_NS = new Set(["__platform__"]); -`); +const sharedNsUrl = repositoryFileUrl("shared/ns-pattern.js"); const controlS3Url = moduleDataUrl(` export async function putAsset() { @@ -253,7 +234,17 @@ export async function resolveDatabaseRefFrom(session, ns, databaseRef) { } `); +const secretEnvelopeUrl = moduleDataUrl(` +export class SecretEnvelopeError extends Error { + constructor(code, message) { + super(message); + this.code = code; + } +} +`); + const controlEnvBudgetUrl = moduleDataUrl(` +import { SecretEnvelopeError } from ${JSON.stringify(secretEnvelopeUrl)}; export class WorkerEnvBudgetError extends Error { constructor(message, details = {}) { super(message); @@ -274,15 +265,12 @@ export function assertWorkerLoaderUserEnvBudget() { } return 0; } -export async function decryptSecretHash() { return {}; } -`); - -const secretEnvelopeUrl = moduleDataUrl(` -export class SecretEnvelopeError extends Error { - constructor(code, message) { - super(message); - this.code = code; +export async function decryptSecretHash() { + const error = /** @type {any} */ (globalThis).__controlDeployTestState.secretEnvelopeError; + if (error) { + throw new SecretEnvelopeError(error.code, error.message); } + return {}; } `); @@ -691,6 +679,73 @@ test("deploy handler resolves cross-namespace service-binding meta from the targ } }); +test("deploy handler classifies empty service target metadata before commit", async () => { + /** @type {any} */ (globalThis).__controlDeployTestState.redis = { + /** @param {string} key @param {string} field */ + async hGet(key, field) { + if (key === "routes:other" && field === "api") return "v1"; + if (key === "worker:other:api:v:1" && field === "__meta__") return ""; + return null; + }, + async incr() { + throw new Error("corrupt target metadata must fail before version allocation"); + }, + }; + + const response = await handle({ + request: new Request("http://control/ns/tenant-a/workers/caller/deploy", { + method: "POST", + body: JSON.stringify({ + mainModule: "worker.js", + modules: { "worker.js": "export default {}" }, + bindings: { + API: { type: "service", ns: "other", service: "api" }, + }, + }), + }), + env: {}, + ns: "tenant-a", + name: "caller", + requestId: "rid-corrupt-service-meta", + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_meta"); + assert.equal(body.message, "Internal error"); + assert.ok(CONTROL_DEPLOY_TEST_STATE.logs.some((/** @type {any} */ entry) => ( + entry.event === "deploy_rejected" && + entry.fields.error_message === "Corrupt __meta__ for other/api/v1" + ))); +}); + +test("deploy handler fails closed instead of hiding empty platform export metadata", async () => { + installPlatformAuthWarningFixture(); + /** @param {string} key @param {string} field */ + const corruptPlatformMetaHGet = async (key, field) => { + if (key === "worker:__platform__:auth:v:1" && field === "__meta__") return ""; + return null; + }; + /** @type {any} */ (globalThis).__controlDeployTestState.redis.hGet = corruptPlatformMetaHGet; + + const response = await handle({ + request: new Request("http://control/ns/tenant-a/workers/caller/deploy", { + method: "POST", + body: JSON.stringify({ + mainModule: "worker.js", + modules: { "worker.js": "export default {}" }, + }), + }), + env: {}, + ns: "tenant-a", + name: "caller", + requestId: "rid-corrupt-platform-meta", + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_meta"); + assert.equal(body.message, "Internal error"); +}); + test("deploy handler schedules cleanup when the first asset upload fails", async () => { /** @type {any} */ (globalThis).__controlDeployTestState.strings = new Map(); /** @type {any} */ (globalThis).__controlDeployTestState.hashes = new Map(); @@ -742,7 +797,10 @@ test("deploy handler schedules cleanup when the first asset upload fails", async requestId: "rid-asset-upload-fail", }); - assert.equal(response.status, 502); + const body = await readJsonResponse(response, 502); + assert.equal(body.error, "asset_upload_failed"); + assert.equal(body.message, "Asset upload failed"); + assert.doesNotMatch(JSON.stringify(body), /simulated upload failure/); assert.deepEqual(/** @type {any} */ (globalThis).__controlDeployTestState.putAssetCalls[0].slice(0, 2), [ {}, "assets/demo/style.css", @@ -920,6 +978,186 @@ test("deploy handler preserves warnings on commit env-budget rejection", async ( } }); +test("deploy handler filters diagnostic aliases from rejection logs", async () => { + /** @type {any} */ (globalThis).__controlDeployTestState.strings = new Map(); + /** @type {any} */ (globalThis).__controlDeployTestState.hashes = new Map(); + /** @type {any} */ (globalThis).__controlDeployTestState.execFailures = 10; + /** @type {any} */ (globalThis).__controlDeployTestState.redis = { + async hKeys() { + return []; + }, + async hGetAll() { + return {}; + }, + async hGet() { + return null; + }, + async incr() { + return 1; + }, + /** @param {(s: ReturnType) => Promise} fn */ + async session(fn) { + return await fn(makeSession()); + }, + }; + + try { + const response = await handle({ + request: new Request("http://control/ns/tenant-a/workers/contention/deploy", { + method: "POST", + body: JSON.stringify({ + mainModule: "worker.js", + modules: { "worker.js": "export default {}" }, + }), + }), + env: {}, + ns: "tenant-a", + name: "contention", + requestId: "rid-deploy-contention", + }); + + const body = await readJsonResponse(response, 503); + assert.equal(body.error, "deploy_contention"); + const rejection = CONTROL_DEPLOY_TEST_STATE.logs.find((/** @type {any} */ entry) => + entry.event === "deploy_rejected" + ); + assert.equal(rejection.fields.version, "v1"); + assert.equal(rejection.fields.error_message, "exhausted 5 retries; retry later"); + assert.equal(Object.hasOwn(rejection.fields, "message"), false); + } finally { + /** @type {any} */ (globalThis).__controlDeployTestState.redis = null; + } +}); + +test("deploy handler keeps D1 ids camelCase on the wire and snake_case in logs", async () => { + /** @type {any} */ (globalThis).__controlDeployTestState.strings = new Map(); + /** @type {any} */ (globalThis).__controlDeployTestState.hashes = new Map(); + /** @type {any} */ (globalThis).__controlDeployTestState.redis = { + async hKeys() { + return []; + }, + async hGetAll() { + return {}; + }, + async hGet() { + return null; + }, + async incr() { + return 2; + }, + /** @param {(s: ReturnType) => Promise} fn */ + async session(fn) { + return await fn(makeSession()); + }, + }; + + try { + const response = await handle({ + request: new Request("http://control/ns/tenant-a/workers/d1-missing/deploy", { + method: "POST", + body: JSON.stringify({ + mainModule: "worker.js", + modules: { "worker.js": "export default {}" }, + bindings: { + DB: { type: "d1", databaseId: "missing-db" }, + }, + }), + }), + env: {}, + ns: "tenant-a", + name: "d1-missing", + requestId: "rid-deploy-d1-missing", + }); + + const body = await readJsonResponse(response, 404); + assert.equal(body.error, "d1_database_not_found"); + assert.equal(body.databaseId, "missing-db"); + const rejection = CONTROL_DEPLOY_TEST_STATE.logs.find((/** @type {any} */ entry) => + entry.event === "deploy_rejected" + ); + assert.equal(rejection.fields.database_id, "missing-db"); + assert.equal(Object.hasOwn(rejection.fields, "databaseId"), false); + } finally { + /** @type {any} */ (globalThis).__controlDeployTestState.redis = null; + } +}); + +test("deploy handler hides secret provider diagnostics and logs them", async () => { + /** @type {any} */ (globalThis).__controlDeployTestState.strings = new Map(); + /** @type {any} */ (globalThis).__controlDeployTestState.hashes = new Map(); + /** @type {any} */ (globalThis).__controlDeployTestState.preparedBundle = { + meta: { + mainModule: "worker.js", + modules: { "worker.js": { type: "module" } }, + }, + normalized: [["worker.js", "export default {}"]], + }; + /** @type {any} */ (globalThis).__controlDeployTestState.secretEnvelopeError = { + code: "secret_encryption_unconfigured", + message: "SECRET_ENVELOPE_LOCAL_KEY_B64 must decode to 32 bytes", + }; + + installPlatformAuthWarningFixture({ + async incr() { + return 8; + }, + /** @param {(s: ReturnType) => Promise} fn */ + async session(fn) { + return await fn(makeSession()); + }, + }); + + try { + const response = await handle({ + request: new Request("http://control/ns/tenant-a/workers/secret-error/deploy", { + method: "POST", + body: JSON.stringify({ + mainModule: "worker.js", + modules: { "worker.js": "export default {}" }, + }), + }), + env: {}, + ns: "tenant-a", + name: "secret-error", + requestId: "rid-secret-provider", + }); + + const body = await readJsonResponse(response, 503); + assert.equal(body.error, "secret_encryption_unconfigured"); + assert.equal(body.message, "Internal error"); + assert.equal(JSON.stringify(body).includes("SECRET_ENVELOPE_LOCAL_KEY_B64"), false); + assert.deepEqual(body.warnings, [{ + binding: "AUTH", + platform: "auth", + missingCallerSecrets: ["API_TOKEN"], + }]); + assert.deepEqual( + CONTROL_DEPLOY_TEST_STATE.logs.find((/** @type {any} */ entry) => + entry.event === "deploy_rejected" + ), + { + level: "error", + event: "deploy_rejected", + fields: { + request_id: "rid-secret-provider", + namespace: "tenant-a", + worker: "secret-error", + version: "v8", + status: 503, + reason: "secret_encryption_unconfigured", + error_message: "SECRET_ENVELOPE_LOCAL_KEY_B64 must decode to 32 bytes", + error_detail: "SECRET_ENVELOPE_LOCAL_KEY_B64 must decode to 32 bytes", + }, + }, + ); + } finally { + /** @type {any} */ (globalThis).__controlDeployTestState.redis = null; + /** @type {any} */ (globalThis).__controlDeployTestState.preparedBundle = null; + /** @type {any} */ (globalThis).__controlDeployTestState.parsedPlatformBindings = null; + /** @type {any} */ (globalThis).__controlDeployTestState.secretEnvelopeError = null; + } +}); + test("deploy handler rejects runtime reserved module collisions before version allocation", async () => { /** @type {any} */ (globalThis).__controlDeployTestState.strings = new Map(); /** @type {any} */ (globalThis).__controlDeployTestState.hashes = new Map(); @@ -1302,6 +1540,60 @@ test("commitWithWatch assigns stable workflow keys into bundle meta and wf:defs" ]); }); +test("commitWithWatch reads only workflow definitions declared by the new bundle", async () => { + const duplicateKey = "wf_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"; + /** @type {unknown[][]} */ + const workflowDefReads = []; + /** @type {any} */ (globalThis).__controlDeployTestState.strings = new Map(); + /** @type {any} */ (globalThis).__controlDeployTestState.hashes = new Map([ + ["wf:defs:tenant-a:orders", { + orders: JSON.stringify({ workflowKey: duplicateKey, className: "OrderWorkflow" }), + historical: "not-json", + }], + ]); + /** @type {any} */ (globalThis).__controlDeployTestState.stagedMeta = null; + /** @type {any} */ (globalThis).__controlDeployTestState.watchedKeys = []; + /** @type {any} */ (globalThis).__controlDeployTestState.execFailures = 0; + + const redis = { + /** @param {(s: ReturnType) => Promise} fn */ + async session(fn) { + const session = makeSession(); + const hMGet = session.hMGet.bind(session); + session.hMGet = async (key, fields) => { + workflowDefReads.push([key, [...fields]]); + return await hMGet(key, fields); + }; + return await fn(session); + }, + }; + + await commitWithWatch({ + redis, + ns: "tenant-a", + name: "orders", + version: "v1", + prepared: { + meta: { + mainModule: "worker.js", + modules: { "worker.js": { type: "esm" } }, + workflows: [ + { name: "orders", binding: "ORDERS", className: "OrderWorkflow" }, + ], + }, + normalized: [["worker.js", "export default {}"]], + }, + outgoingRefs: [], + d1Refs: [], + controlEnv: {}, + }); + assert.equal( + /** @type {any} */ (globalThis).__controlDeployTestState.stagedMeta.workflows[0].workflowKey, + duplicateKey + ); + assert.deepEqual(workflowDefReads, [["wf:defs:tenant-a:orders", ["orders"]]]); +}); + test("commitWithWatch checks code budget after workflow keys are materialized", async () => { /** @type {any} */ (globalThis).__controlDeployTestState.strings = new Map(); /** @type {any} */ (globalThis).__controlDeployTestState.hashes = new Map([ @@ -1421,6 +1713,40 @@ test("commitWithWatch reads workflow defs with own-property discipline", async ( ]); }); +/** @param {any} redis */ +function platformBindingCommitArgs(redis) { + return { + redis, + ns: "tenant-a", + name: "caller", + version: "v1", + prepared: { + meta: { + mainModule: "worker.js", + modules: { "worker.js": { type: "esm" } }, + bindings: { + PLATFORM: { + type: "service", + ns: "__platform__", + service: "platformApi", + version: "v1", + entrypoint: "Api", + }, + }, + }, + normalized: [["worker.js", "export default {}"]], + }, + outgoingRefs: [{ + binding: "PLATFORM", + targetNs: "__platform__", + targetWorker: "platformApi", + targetVersion: "v1", + }], + d1Refs: [], + controlEnv: {}, + }; +} + test("commitWithWatch rejects platform binding target drift before commit", async () => { /** @type {any} */ (globalThis).__controlDeployTestState.strings = new Map(); /** @type {any} */ (globalThis).__controlDeployTestState.hashes = new Map([ @@ -1444,36 +1770,7 @@ test("commitWithWatch rejects platform binding target drift before commit", asyn }; await assert.rejects( - () => commitWithWatch({ - redis, - ns: "tenant-a", - name: "caller", - version: "v1", - prepared: { - meta: { - mainModule: "worker.js", - modules: { "worker.js": { type: "esm" } }, - bindings: { - PLATFORM: { - type: "service", - ns: "__platform__", - service: "platformApi", - version: "v1", - entrypoint: "Api", - }, - }, - }, - normalized: [["worker.js", "export default {}"]], - }, - outgoingRefs: [{ - binding: "PLATFORM", - targetNs: "__platform__", - targetWorker: "platformApi", - targetVersion: "v1", - }], - d1Refs: [], - controlEnv: {}, - }), + () => commitWithWatch(platformBindingCommitArgs(redis)), (err) => { const deployErr = /** @type {any} */ (err); assert.equal(deployErr.code, "target_drift"); @@ -1490,3 +1787,38 @@ test("commitWithWatch rejects platform binding target drift before commit", asyn assert.ok(/** @type {any} */ (globalThis).__controlDeployTestState.watchedKeys.includes("worker:__platform__:platformApi:v:1")); assert.ok(/** @type {any} */ (globalThis).__controlDeployTestState.watchedKeys.includes("worker-delete-lock:__platform__:platformApi")); }); + +test("commitWithWatch rejects empty platform binding target metadata before commit", async () => { + /** @type {any} */ (globalThis).__controlDeployTestState.strings = new Map(); + /** @type {any} */ (globalThis).__controlDeployTestState.hashes = new Map([ + ["routes:__platform__", { platformApi: "v1" }], + ["worker:__platform__:platformApi:v:1", { "__meta__": "" }], + ]); + /** @type {any} */ (globalThis).__controlDeployTestState.stagedMeta = null; + /** @type {any} */ (globalThis).__controlDeployTestState.watchedKeys = []; + /** @type {any} */ (globalThis).__controlDeployTestState.execFailures = 0; + + const redis = { + /** @param {(s: ReturnType) => Promise} fn */ + async session(fn) { + return await fn(makeSession()); + }, + }; + + await assert.rejects( + () => commitWithWatch(platformBindingCommitArgs(redis)), + (err) => { + const deployErr = /** @type {any} */ (err); + assert.equal(deployErr.code, "target_drift"); + assert.equal(deployErr.details.target.ns, "__platform__"); + assert.equal(deployErr.details.target.worker, "platformApi"); + assert.equal(deployErr.details.target.version, "v1"); + assert.equal(deployErr.details.target.reason, "bundle_missing"); + return true; + } + ); + + assert.equal(/** @type {any} */ (globalThis).__controlDeployTestState.stagedMeta, null); + assert.ok(/** @type {any} */ (globalThis).__controlDeployTestState.watchedKeys.includes("worker:__platform__:platformApi:v:1")); + assert.ok(/** @type {any} */ (globalThis).__controlDeployTestState.watchedKeys.includes("worker-delete-lock:__platform__:platformApi")); +}); diff --git a/tests/unit/control-env-budget.test.js b/tests/unit/control-env-budget.test.js index 20dd10c..ec1baa2 100644 --- a/tests/unit/control-env-budget.test.js +++ b/tests/unit/control-env-budget.test.js @@ -3,26 +3,29 @@ import assert from "node:assert/strict"; import { importRepositoryModule, importSpecifierReplacements, - moduleDataUrl, repositoryFileUrl, + repositoryModuleDataUrl, } from "../helpers/load-shared-module.js"; +import { sharedRedisStubUrl } from "../helpers/mocks/fake-redis.js"; +import { compileControlGraph } from "../helpers/load-control-lib.js"; import { encryptSecretValue } from "../../shared/secret-envelope.js"; +const { libUrl: controlLibUrl } = await compileControlGraph(); const secretEnvelopeUrl = repositoryFileUrl("shared/secret-envelope.js"); -const sharedErrorsUrl = repositoryFileUrl("shared/errors.js"); const sharedVersionUrl = repositoryFileUrl("shared/version.js"); -const sharedRedisUrl = moduleDataUrl(` -export class WatchError extends Error { - constructor(message = "watched key changed") { - super(message); - this.name = "WatchError"; - } -} -`); +const sharedRedisUrl = sharedRedisStubUrl(); +const runtimeEnvBuildUrl = repositoryModuleDataUrl( + "runtime/load/env-build.js", + importSpecifierReplacements({ + "shared-ns-pattern": repositoryFileUrl("shared/ns-pattern.js"), + "shared-version": repositoryFileUrl("shared/version.js"), + }) +); const { UPSTREAM_WORKER_LOADER_ENV_MAX_BYTES, WORKER_LOADER_ENV_HEADROOM_BYTES, WORKER_LOADER_ENV_MAX_BYTES, + BundleMetaError, WorkerEnvBudgetError, assertWorkerLoaderUserEnvBudget, assertWorkerVersionsUserEnvBudget, @@ -31,8 +34,9 @@ const { estimatedWorkerLoaderEnvBytes, WORKER_LOADER_ENV_VERSION_PLACEHOLDER, } = await importRepositoryModule("control/env-budget.js", importSpecifierReplacements({ + "control-lib": controlLibUrl, + "runtime-load-env-build": runtimeEnvBuildUrl, "shared-secret-envelope": secretEnvelopeUrl, - "shared-errors": sharedErrorsUrl, "shared-version": sharedVersionUrl, "shared-redis": sharedRedisUrl, })); @@ -42,6 +46,19 @@ const envelopeEnv = { SECRET_ENVELOPE_KID: "local:test:secret-envelope:v1", }; +/** @param {unknown} err @param {string} reason */ +function assertBundleMetaError(err, reason) { + assert.ok(err instanceof BundleMetaError); + const bundleError = /** @type {Error & { status: number, code: string, details: Record, cause: unknown }} */ (err); + assert.equal(bundleError.status, 500); + assert.equal(bundleError.code, "corrupt_meta"); + assert.equal(bundleError.message, "Corrupt __meta__ for demo/api/v1"); + assert.deepEqual(bundleError.details, { namespace: "demo", worker: "api", version: "v1" }); + assert.ok(bundleError.cause instanceof Error); + assert.equal(bundleError.cause.message, reason); + return true; +} + test("worker env budget counts merged vars and secrets with worker-secret precedence", () => { const estimated = estimatedWorkerLoaderEnv({ ns: "demo", @@ -166,11 +183,11 @@ test("worker env budget accounts for V8 two-byte strings on mixed non-Latin-1 en ); }); -test("worker env budget stores required caller secrets in a null-prototype map", () => { +test("worker env budget stores required caller secrets in a JSRPC-serializable object", () => { const estimated = estimatedWorkerLoaderEnv({ ns: "demo", worker: "caller", - nsSecrets: JSON.parse('{"__proto__":"secret"}'), + nsSecrets: { API_TOKEN: "secret" }, meta: { bindings: { PLATFORM: { @@ -178,16 +195,15 @@ test("worker env budget stores required caller secrets in a null-prototype map", ns: "__platform__", service: "platformApi", version: "v1", - requiredCallerSecrets: ["__proto__"], + requiredCallerSecrets: ["API_TOKEN"], }, }, }, }); const callerSecrets = /** @type {any} */ (estimated.PLATFORM).props.callerSecrets; - assert.equal(Object.getPrototypeOf(callerSecrets), null); - assert.equal(Object.hasOwn(callerSecrets, "__proto__"), true); - assert.equal(callerSecrets.__proto__, "secret"); + assert.equal(Object.getPrototypeOf(callerSecrets), Object.prototype); + assert.deepEqual(callerSecrets, { API_TOKEN: "secret" }); }); test("worker env budget counts configured assets CDN base", () => { @@ -471,7 +487,35 @@ test("worker env budget reports bundle metadata parse context", async () => { worker: "api", versions: ["v1"], }), - /invalid bundle metadata for demo\/api@v1/ + (err) => { + assert.ok(err instanceof BundleMetaError); + const bundleError = /** @type {Error & { cause: unknown }} */ (err); + assert.ok(bundleError.cause instanceof SyntaxError); + return assertBundleMetaError(err, bundleError.cause.message); + } + ); +}); + +test("worker env budget maps strict retained metadata failures to corrupt_meta", async () => { + const redis = { + async hGet() { + return JSON.stringify({ + workflows: [{ binding: "FLOW", name: "flow", className: "Flow" }], + }); + }, + }; + + await assert.rejects( + () => assertWorkerVersionsUserEnvBudget({ + redis, + ns: "demo", + worker: "api", + versions: ["v1"], + }), + (err) => assertBundleMetaError( + err, + 'Workflow binding "FLOW" is missing workflow metadata (redeploy demo/api)' + ) ); }); @@ -492,7 +536,7 @@ test("worker env budget fails closed when retained bundle metadata is missing", worker: "api", versions: ["v1"], }), - /bundle metadata missing for demo\/api@v1/ + (err) => assertBundleMetaError(err, "__meta__ must be a JSON string") ); }); @@ -535,6 +579,6 @@ test("worker env budget fails closed when retained bundle metadata is not an obj worker: "api", versions: ["v1"], }), - /__meta__ must be a JSON object/ + (err) => assertBundleMetaError(err, "__meta__ must be a JSON object") ); }); diff --git a/tests/unit/control-handlers-workers.test.js b/tests/unit/control-handlers-workers.test.js index 8da5ebd..f6b02cf 100644 --- a/tests/unit/control-handlers-workers.test.js +++ b/tests/unit/control-handlers-workers.test.js @@ -11,6 +11,7 @@ import { readRepositoryFile, repositoryFileUrl, } from "../helpers/load-shared-module.js"; +import { compileControlGraph } from "../helpers/load-control-lib.js"; import { createFakeRedis } from "../helpers/mocks/fake-redis.js"; import { assertJsonResponse } from "../helpers/response-json.js"; @@ -23,12 +24,7 @@ const workersHandlerState = installControlHandlerState( }) ); const controlSharedUrl = controlSharedHarnessUrl(WORKERS_HANDLER_STATE_GLOBAL); - -const controlLibUrl = moduleDataUrl(` -export function routesKey(ns) { return "routes:" + ns; } -export function workersIndexKey(ns) { return "workers:" + ns; } -export function workerVersionsKey(ns, name) { return "worker-versions:" + ns + ":" + name; } -`); +const { libUrl: controlLibUrl } = await compileControlGraph(); const sharedSecretKeysUrl = repositoryFileUrl("shared/secret-keys.js"); @@ -42,7 +38,7 @@ const { handle } = await import(moduleDataUrl(src)); function resetWorkersHandlerState() { const redis = createFakeRedis(); - redis.sets.set("workers:demo", new Set(["beta", "alpha"])); + redis.sets.set("workers:demo", new Set(["gamma", "beta", "alpha"])); redis.hashes.set("routes:demo", { alpha: "v2" }); redis.zsets.set("worker-versions:demo:alpha", new Map([ ["v1", 1], @@ -50,6 +46,7 @@ function resetWorkersHandlerState() { ])); redis.zsets.set("worker-versions:demo:beta", new Map([["v1", 1]])); redis.hashes.set("secrets:demo:beta", { TOKEN: "WDL-ENC:test" }); + redis.hashes.set("wf:defs:demo:gamma", { flow: "{}" }); let sessions = 0; const session = redis.session.bind(redis); redis.session = async (fn) => { @@ -76,6 +73,7 @@ test("workers handler lists namespace state through one Redis session", async () versions: ["v1", "v2"], versionCount: 2, hasSecrets: false, + hasWorkflowDefs: false, }, { name: "beta", @@ -83,6 +81,15 @@ test("workers handler lists namespace state through one Redis session", async () versions: ["v1"], versionCount: 1, hasSecrets: true, + hasWorkflowDefs: false, + }, + { + name: "gamma", + activeVersion: null, + versions: [], + versionCount: 0, + hasSecrets: false, + hasWorkflowDefs: true, }, ], }); @@ -92,15 +99,20 @@ test("workers handler lists namespace state through one Redis session", async () ["zRangeMany", [ "worker-versions:demo:alpha", "worker-versions:demo:beta", + "worker-versions:demo:gamma", ], 0, -1], ["existsMany", [ "secrets:demo:alpha", "secrets:demo:beta", + "secrets:demo:gamma", + "wf:defs:demo:alpha", + "wf:defs:demo:beta", + "wf:defs:demo:gamma", ]], ]); assert.deepEqual(state.logs, [{ level: "info", event: "workers_listed", - fields: { request_id: "rid-workers", namespace: "demo", count: 2 }, + fields: { request_id: "rid-workers", namespace: "demo", count: 3 }, }]); }); diff --git a/tests/unit/control-handlers-workflows.test.js b/tests/unit/control-handlers-workflows.test.js index ca23002..cdd6ef8 100644 --- a/tests/unit/control-handlers-workflows.test.js +++ b/tests/unit/control-handlers-workflows.test.js @@ -6,41 +6,28 @@ import { installControlHandlerState, } from "../helpers/control-handler-harness.js"; import { - moduleDataUrl, + repositoryFileUrl, } from "../helpers/load-shared-module.js"; +import { compileControlGraph } from "../helpers/load-control-lib.js"; import { parseJsonObjectRequestBody } from "../helpers/request-body.js"; -import { assertJsonResponse } from "../helpers/response-json.js"; +import { assertJsonResponse, readJsonResponse } from "../helpers/response-json.js"; import { sharedInternalAuthUrl } from "../helpers/runtime-proxy-stub.js"; const TEST_INTERNAL_AUTH_TOKEN = "test-internal-auth-token"; const WORKFLOWS_HANDLER_GLOBAL = "__workflowsHandlerState"; - -const controlLibUrl = moduleDataUrl(` -export const WORKER_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9_-]*$/; -export const WORKFLOW_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9_-]*$/; -export function isValidWorkerName(name) { return typeof name === "string" && WORKER_NAME_RE.test(name); } -export function isValidWorkflowName(name) { return typeof name === "string" && WORKFLOW_NAME_RE.test(name); } -export function workflowDefsKey(ns, worker) { return "wf:defs:" + ns + ":" + worker; } -`); - -const sharedVersionUrl = moduleDataUrl(` -export function parseVersion(tag) { - const match = /^v([1-9][0-9]*)$/.exec(tag); - return match ? Number(match[1]) : null; -} -export function bundleKey(ns, worker, version) { - const n = parseVersion(version); - if (n == null) throw new Error("invalid version tag " + JSON.stringify(version)); - return "worker:" + ns + ":" + worker + ":v:" + n; -} -export function routesKey(ns) { return "routes:" + ns; } -`); +const ACTIVE_WORKFLOW_KEY = `wf_${"1".repeat(32)}`; +const NEW_WORKFLOW_KEY = `wf_${"2".repeat(32)}`; +const RETIRED_WORKFLOW_KEY = `wf_${"3".repeat(32)}`; +const { libUrl: productionControlLibUrl } = await compileControlGraph(); +const sharedVersionUrl = repositoryFileUrl("shared/version.js"); +const sharedNsPatternUrl = repositoryFileUrl("shared/ns-pattern.js"); const { handle } = await importControlHandler("control/handlers/workflows.js", { globalName: WORKFLOWS_HANDLER_GLOBAL, replacements: { - "control-lib": controlLibUrl, + "control-lib": productionControlLibUrl, "shared-internal-auth": sharedInternalAuthUrl(), + "shared-ns-pattern": sharedNsPatternUrl, "shared-version": sharedVersionUrl, }, }); @@ -51,7 +38,7 @@ function resetWorkflowsHandlerState() { name: "orders", binding: "ORDERS", className: "OrderWorkflow", - workflowKey: "wf_1234", + workflowKey: ACTIVE_WORKFLOW_KEY, }], }); const state = createControlHandlerState({ @@ -61,8 +48,9 @@ function resetWorkflowsHandlerState() { redis.hashes.set("routes:demo", { api: "v2" }); redis.hashes.set("worker:demo:api:v:2", { "__meta__": meta }); state.workflows = { - /** @param {string} url @param {{ body: string, headers?: HeadersInit }} init */ + /** @param {string} url @param {{ body: string, headers?: HeadersInit, signal?: AbortSignal | null }} init */ async fetch(url, init) { + assert.equal(init.signal, undefined); assert.equal(new Headers(init.headers).get("x-wdl-internal-auth"), TEST_INTERNAL_AUTH_TOKEN); redis.commands.push(["fetch", url, parseJsonObjectRequestBody(init, "workflows backend request body")]); return new Response(JSON.stringify({ id: "order-1", status: "paused" }), { @@ -95,13 +83,14 @@ test("workflows handler lists active workflow definitions from bundle metadata", name: "orders", binding: "ORDERS", className: "OrderWorkflow", - workflowKey: "wf_1234", + workflowKey: ACTIVE_WORKFLOW_KEY, }], }); assert.deepEqual(state.redis.commands, [ ["hGetAll", "routes:demo"], ["hGetMany", [["worker:demo:api:v:2", "__meta__"]]], ["hGetAllMany", ["wf:defs:demo:api"]], + ["hGetAll", "routes:demo"], ]); assert.deepEqual(state.logs, [{ level: "info", @@ -110,6 +99,354 @@ test("workflows handler lists active workflow definitions from bundle metadata", }]); }); +for (const [label, rawMeta] of [ + ["missing", null], + ["empty", ""], + ["malformed", "SECRET_TOKEN_ABC"], + ["non-object", "[]"], +]) { + test(`workflows handler fails closed on ${label} active bundle metadata`, async () => { + const state = resetWorkflowsHandlerState(); + state.redis.hashes.set( + "worker:demo:api:v:2", + rawMeta === null ? {} : { "__meta__": rawMeta } + ); + + const response = await handle({ + method: "GET", + url: new URL("http://control/ns/demo/workflows"), + ns: "demo", + subPath: [], + requestId: `rid-${label}-meta`, + }); + + await assertJsonResponse(response, 500, { + namespace: "demo", + worker: "api", + version: "v2", + error: "corrupt_meta", + message: "Internal error", + }); + const rejection = state.logs.find((/** @type {any} */ entry) => + entry.event === "workflow_request_rejected" + ); + assert.ok(rejection); + assert.equal(rejection.level, "error"); + assert.equal(rejection.fields.request_id, `rid-${label}-meta`); + assert.equal(rejection.fields.namespace, "demo"); + assert.equal(rejection.fields.worker, "api"); + assert.equal(rejection.fields.status, 500); + assert.equal(rejection.fields.reason, "corrupt_meta"); + assert.equal(rejection.fields.error_message, "Corrupt __meta__ for demo/api/v2"); + assert.equal(rejection.fields.metadata_version, "v2"); + assert.equal(rejection.fields.stage, "bundle_meta_parse"); + assert.equal(typeof rejection.fields.error_detail, "string"); + if (label === "malformed") { + assert.equal(rejection.fields.error_detail, "__meta__ is not valid JSON"); + assert.equal(JSON.stringify(state.logs).includes(String(rawMeta)), false); + } + }); +} + +test("workflows handler fails closed on malformed active workflow entries", async () => { + const state = resetWorkflowsHandlerState(); + state.redis.hashes.set("worker:demo:api:v:2", { + "__meta__": JSON.stringify({ + workflows: [{ + name: "orders", + binding: "ORDERS", + className: "OrderWorkflow", + }], + }), + }); + + const response = await handle({ + method: "GET", + url: new URL("http://control/ns/demo/workflows"), + ns: "demo", + subPath: [], + requestId: "rid-malformed-workflow-entry", + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_meta"); + assert.equal(body.stage, undefined); + const rejection = state.logs.find((/** @type {any} */ entry) => + entry.event === "workflow_request_rejected" + ); + assert.equal(rejection.fields.stage, "workflow_entries_parse"); +}); + +test("workflows handler rejects active workflows that share a workflow key", async () => { + const state = resetWorkflowsHandlerState(); + state.redis.hashes.set("worker:demo:api:v:2", { + "__meta__": JSON.stringify({ + workflows: [ + { + name: "orders", + binding: "ORDERS", + className: "OrderWorkflow", + workflowKey: ACTIVE_WORKFLOW_KEY, + }, + { + name: "billing", + binding: "BILLING", + className: "BillingWorkflow", + workflowKey: ACTIVE_WORKFLOW_KEY, + }, + ], + }), + }); + + const response = await handle({ + method: "GET", + url: new URL("http://control/ns/demo/workflows"), + ns: "demo", + subPath: [], + requestId: "rid-duplicate-active-workflow-key", + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_meta"); + const rejection = state.logs.find((/** @type {any} */ entry) => + entry.event === "workflow_request_rejected" + ); + assert.equal(rejection.fields.stage, "workflow_entries_parse"); +}); + +test("workflows handler fails closed on malformed persisted workflow definitions", async () => { + const state = resetWorkflowsHandlerState(); + state.redis.hashes.set("wf:defs:demo:api", { retired: "not-json" }); + + const response = await handle({ + method: "GET", + url: new URL("http://control/ns/demo/workflows"), + ns: "demo", + subPath: [], + requestId: "rid-malformed-workflow-def", + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_meta"); + const rejection = state.logs.find((/** @type {any} */ entry) => + entry.event === "workflow_request_rejected" + ); + assert.equal(rejection.fields.stage, "workflow_defs_parse"); +}); + +test("workflows handler retries a list snapshot split by whole-worker delete", async () => { + const state = resetWorkflowsHandlerState(); + const redis = /** @type {any} */ (state.redis); + redis.hGetMany = async (/** @type {Array<[string, string]>} */ pairs) => { + redis.commands.push(["hGetMany", pairs]); + redis.hashes.set("routes:demo", {}); + redis.hashes.set("worker:demo:api:v:2", {}); + return pairs.map(() => null); + }; + + const response = await handle({ + method: "GET", + url: new URL("http://control/ns/demo/workflows"), + ns: "demo", + subPath: [], + requestId: "rid-list-delete-race", + }); + + await assertJsonResponse(response, 200, { + namespace: "demo", + workflows: [], + }); + assert.deepEqual(state.logs, [{ + level: "info", + event: "workflows_listed", + fields: { request_id: "rid-list-delete-race", namespace: "demo", count: 0 }, + }]); +}); + +test("workflows handler returns worker_not_found when whole-delete wins metadata resolution", async () => { + const state = resetWorkflowsHandlerState(); + const redis = /** @type {any} */ (state.redis); + const originalHGet = redis.hGet.bind(redis); + redis.hGet = async (/** @type {string} */ key, /** @type {string} */ field) => { + if (key === "worker:demo:api:v:2" && field === "__meta__") { + redis.commands.push(["hGet", key, field]); + redis.hashes.set("routes:demo", {}); + redis.hashes.set("worker:demo:api:v:2", {}); + return null; + } + return await originalHGet(key, field); + }; + + const response = await handle({ + method: "GET", + url: new URL("http://control/ns/demo/workflows/api/orders/instances/order-1"), + ns: "demo", + subPath: ["api", "orders", "instances", "order-1"], + requestId: "rid-resolve-delete-race", + }); + + await assertJsonResponse(response, 404, { + error: "worker_not_found", + message: "Worker demo/api is not active", + }); + assert.equal(redis.commands.some((/** @type {unknown[]} */ command) => command[0] === "fetch"), false); +}); + +test("workflows handler returns contention when workflow resolution never stabilizes", async () => { + const state = resetWorkflowsHandlerState(); + const redis = /** @type {any} */ (state.redis); + const originalHGet = redis.hGet.bind(redis); + const nextVersions = new Map([ + ["worker:demo:api:v:2", "v3"], + ["worker:demo:api:v:3", "v4"], + ]); + redis.hGet = async (/** @type {string} */ key, /** @type {string} */ field) => { + const nextVersion = field === "__meta__" ? nextVersions.get(key) : undefined; + if (nextVersion) { + redis.commands.push(["hGet", key, field]); + redis.hashes.set("routes:demo", { api: nextVersion }); + return null; + } + return await originalHGet(key, field); + }; + + const response = await handle({ + method: "GET", + url: new URL("http://control/ns/demo/workflows/api/orders/instances/order-1"), + ns: "demo", + subPath: ["api", "orders", "instances", "order-1"], + requestId: "rid-resolve-contention", + }); + + await assertJsonResponse(response, 503, { + error: "workflow_metadata_contention", + message: "Internal error", + namespace: "demo", + worker: "api", + }); + assert.equal(redis.commands.filter((/** @type {unknown[]} */ command) => + command[0] === "hGet" && String(command[1]).startsWith("worker:demo:api:v:") + ).length, 2); + assert.equal(redis.commands.some((/** @type {unknown[]} */ command) => command[0] === "fetch"), false); +}); + +for (const [label, rawMeta] of [ + ["missing", null], + ["empty", ""], + ["malformed", "SECRET_TOKEN_ABC"], + ["non-object", "[]"], +]) { + test(`workflows handler reports stable ${label} metadata despite sibling route churn`, async () => { + const state = resetWorkflowsHandlerState(); + const redis = /** @type {any} */ (state.redis); + const emptyMeta = JSON.stringify({ workflows: [] }); + redis.hashes.set("worker:demo:api:v:2", rawMeta == null ? {} : { "__meta__": rawMeta }); + redis.hashes.set("worker:demo:billing:v:1", { "__meta__": emptyMeta }); + redis.hashes.set("worker:demo:billing:v:2", { "__meta__": emptyMeta }); + const routeSnapshots = [ + { api: "v2", billing: "v1" }, + { api: "v2", billing: "v2" }, + { api: "v2", billing: "v2" }, + { api: "v2", billing: "v3" }, + ]; + redis.hGetAll = async (/** @type {string} */ key) => { + redis.commands.push(["hGetAll", key]); + if (key === "routes:demo") return routeSnapshots.shift() ?? routeSnapshots.at(-1) ?? {}; + return redis.hashes.get(key) ?? {}; + }; + + const response = await handle({ + method: "GET", + url: new URL("http://control/ns/demo/workflows"), + ns: "demo", + subPath: [], + requestId: `rid-stable-${label}-with-sibling-churn`, + }); + + await assertJsonResponse(response, 500, { + namespace: "demo", + worker: "api", + version: "v2", + error: "corrupt_meta", + message: "Internal error", + }); + assert.equal(routeSnapshots.length, 2); + }); +} + +test("workflows handler returns contention when list routes never stabilize", async () => { + const state = resetWorkflowsHandlerState(); + const redis = /** @type {any} */ (state.redis); + const meta = redis.hashes.get("worker:demo:api:v:2")["__meta__"]; + redis.hashes.set("worker:demo:api:v:3", { "__meta__": meta }); + const routeSnapshots = [ + { api: "v2" }, + { api: "v3" }, + { api: "v3" }, + { api: "v4" }, + ]; + redis.hGetAll = async (/** @type {string} */ key) => { + redis.commands.push(["hGetAll", key]); + if (key === "routes:demo") return routeSnapshots.shift() ?? {}; + return redis.hashes.get(key) ?? {}; + }; + + const response = await handle({ + method: "GET", + url: new URL("http://control/ns/demo/workflows"), + ns: "demo", + subPath: [], + requestId: "rid-list-contention", + }); + + await assertJsonResponse(response, 503, { + error: "workflow_metadata_contention", + message: "Internal error", + namespace: "demo", + }); + assert.equal(routeSnapshots.length, 0); +}); + +test("workflows handler does not combine active metadata with redeployed workflow defs", async () => { + const state = resetWorkflowsHandlerState(); + const redis = /** @type {any} */ (state.redis); + redis.hashes.set("worker:demo:api:v:2", { "__meta__": JSON.stringify({ workflows: [] }) }); + const originalHGet = redis.hGet.bind(redis); + let redeployed = false; + redis.hGet = async (/** @type {string} */ key, /** @type {string} */ field) => { + if (key === "wf:defs:demo:api" && field === "orders" && !redeployed) { + redeployed = true; + redis.hashes.set("routes:demo", { api: "v3" }); + redis.hashes.set("worker:demo:api:v:3", { "__meta__": JSON.stringify({ workflows: [] }) }); + redis.hashes.set(key, { + orders: JSON.stringify({ workflowKey: NEW_WORKFLOW_KEY, className: "NewOrderWorkflow" }), + }); + } + return await originalHGet(key, field); + }; + + const response = await handle({ + method: "GET", + url: new URL("http://control/ns/demo/workflows/api/orders/instances/order-1"), + ns: "demo", + subPath: ["api", "orders", "instances", "order-1"], + requestId: "rid-redeploy-defs-race", + }); + + assert.equal(response.status, 200); + assert.deepEqual(redis.commands.at(-1), ["fetch", "http://workflows/internal/workflows/status", { + ns: "demo", + worker: "api", + frozenVersion: "v3", + workflowName: "orders", + workflowKey: NEW_WORKFLOW_KEY, + className: "NewOrderWorkflow", + instanceId: "order-1", + options: {}, + requestId: "rid-redeploy-defs-race", + }]); +}); + test("workflows handler lists empty namespaces without batch reads", async () => { const state = resetWorkflowsHandlerState(); state.redis.hashes.set("routes:demo", {}); @@ -156,19 +493,32 @@ test("workflows handler preserves metadata-unavailable error shape for batched r await assertJsonResponse(response, 500, { error: "workflow_metadata_unavailable", - message: "Workflow metadata is unavailable", + message: "Internal error", namespace: "demo", worker_count: 2, }); - assert.deepEqual(state.logs, [{ - level: "error", - event: "workflow_metadata_unavailable", - fields: { - namespace: "demo", - worker_count: 2, - error_message: "redis unavailable", + assert.deepEqual(state.logs, [ + { + level: "error", + event: "workflow_metadata_unavailable", + fields: { + namespace: "demo", + worker_count: 2, + error_message: "redis unavailable", + }, }, - }]); + { + level: "error", + event: "workflow_request_rejected", + fields: { + request_id: "rid-meta-fail", + namespace: "demo", + status: 500, + reason: "workflow_metadata_unavailable", + error_message: "Workflow metadata is unavailable", + }, + }, + ]); }); test("workflows handler wraps workflow definition batch read failures", async () => { @@ -193,19 +543,32 @@ test("workflows handler wraps workflow definition batch read failures", async () await assertJsonResponse(response, 500, { error: "workflow_metadata_unavailable", - message: "Workflow metadata is unavailable", + message: "Internal error", namespace: "demo", worker_count: 2, }); - assert.deepEqual(state.logs, [{ - level: "error", - event: "workflow_metadata_unavailable", - fields: { - namespace: "demo", - worker_count: 2, - error_message: "defs unavailable", + assert.deepEqual(state.logs, [ + { + level: "error", + event: "workflow_metadata_unavailable", + fields: { + namespace: "demo", + worker_count: 2, + error_message: "defs unavailable", + }, }, - }]); + { + level: "error", + event: "workflow_request_rejected", + fields: { + request_id: "rid-defs-fail", + namespace: "demo", + status: 500, + reason: "workflow_metadata_unavailable", + error_message: "Workflow metadata is unavailable", + }, + }, + ]); }); test("workflows handler resolves retired workflow definitions from wf:defs", async () => { @@ -214,7 +577,7 @@ test("workflows handler resolves retired workflow definitions from wf:defs", asy state.redis.hashes.set("worker:demo:api:v:3", { "__meta__": JSON.stringify({ workflows: [] }) }); state.redis.hashes.set("wf:defs:demo:api", { orders: JSON.stringify({ - workflowKey: "wf_retired", + workflowKey: RETIRED_WORKFLOW_KEY, className: "OldOrderWorkflow", }), }); @@ -233,7 +596,7 @@ test("workflows handler resolves retired workflow definitions from wf:defs", asy worker: "api", frozenVersion: "v3", workflowName: "orders", - workflowKey: "wf_retired", + workflowKey: RETIRED_WORKFLOW_KEY, className: "OldOrderWorkflow", instanceId: "order-1", requestId: "rid-retired", @@ -246,7 +609,7 @@ test("workflows handler rejects restart for retired workflow definitions", async state.redis.hashes.set("worker:demo:api:v:3", { "__meta__": JSON.stringify({ workflows: [] }) }); state.redis.hashes.set("wf:defs:demo:api", { orders: JSON.stringify({ - workflowKey: "wf_retired", + workflowKey: RETIRED_WORKFLOW_KEY, className: "OldOrderWorkflow", }), }); @@ -302,12 +665,13 @@ test("workflows handler resolves active workflow identity before lifecycle proxy assert.deepEqual(state.redis.commands, [ ["hGet", "routes:demo", "api"], ["hGet", "worker:demo:api:v:2", "__meta__"], + ["hGet", "routes:demo", "api"], ["fetch", "http://workflows/internal/workflows/resume", { ns: "demo", worker: "api", frozenVersion: "v2", workflowName: "orders", - workflowKey: "wf_1234", + workflowKey: ACTIVE_WORKFLOW_KEY, className: "OrderWorkflow", instanceId: "order-1", requestId: "rid-resume", @@ -332,7 +696,7 @@ test("workflows handler forwards status includeSteps options", async () => { worker: "api", frozenVersion: "v2", workflowName: "orders", - workflowKey: "wf_1234", + workflowKey: ACTIVE_WORKFLOW_KEY, className: "OrderWorkflow", instanceId: "order-1", options: { includeSteps: true, stepLimit: 10 }, @@ -354,12 +718,26 @@ test("workflows handler fails closed when workflows backend is unavailable", asy await assertJsonResponse(response, 503, { error: "workflow_internal_dispatch_failed", - message: "Workflow backend is unavailable", + message: "Internal error", }); assert.deepEqual(state.redis.commands, [ ["hGet", "routes:demo", "api"], ["hGet", "worker:demo:api:v:2", "__meta__"], + ["hGet", "routes:demo", "api"], ]); + assert.deepEqual(state.logs, [{ + level: "error", + event: "workflow_request_rejected", + fields: { + request_id: "rid-down", + namespace: "demo", + worker: "api", + workflow: "orders", + status: 503, + reason: "workflow_internal_dispatch_failed", + error_message: "Workflow backend is unavailable", + }, + }]); }); test("workflows handler hides backend 5xx messages but logs diagnostics", async () => { @@ -388,9 +766,11 @@ test("workflows handler hides backend 5xx messages but logs diagnostics", async await assertJsonResponse(response, 500, { upstream_status: 500, error: "redis_error", - message: "Workflow backend request failed", + message: "Internal error", }); - assert.deepEqual(state.logs.at(-1), { + assert.deepEqual(state.logs.find((/** @type {any} */ entry) => + entry.event === "workflow_backend_error" + ), { level: "error", event: "workflow_backend_error", fields: { @@ -421,9 +801,11 @@ test("workflows handler hides backend fetch exceptions but logs diagnostics", as await assertJsonResponse(response, 503, { error: "workflow_internal_dispatch_failed", - message: "Workflow backend request failed", + message: "Internal error", }); - assert.deepEqual(state.logs.at(-1), { + assert.deepEqual(state.logs.find((/** @type {any} */ entry) => + entry.event === "workflow_backend_request_failed" + ), { level: "error", event: "workflow_backend_request_failed", fields: { @@ -452,6 +834,7 @@ test("workflows handler rejects invalid status query options before backend disp assert.deepEqual(state.redis.commands, [ ["hGet", "routes:demo", "api"], ["hGet", "worker:demo:api:v:2", "__meta__"], + ["hGet", "routes:demo", "api"], ]); }); @@ -473,5 +856,6 @@ test("workflows handler rejects snake_case status query options", async () => { assert.deepEqual(state.redis.commands, [ ["hGet", "routes:demo", "api"], ["hGet", "worker:demo:api:v:2", "__meta__"], + ["hGet", "routes:demo", "api"], ]); }); diff --git a/tests/unit/control-index.test.js b/tests/unit/control-index.test.js index fdbb51c..457e405 100644 --- a/tests/unit/control-index.test.js +++ b/tests/unit/control-index.test.js @@ -4,15 +4,17 @@ import { importRepositoryModule, importSpecifierReplacements, moduleDataUrl, + repositoryFileUrl, } from "../helpers/load-shared-module.js"; import { assertJsonResponse, readJsonResponse } from "../helpers/response-json.js"; /** @type {any} */ (globalThis).__controlIndexState = null; +const sharedNsPatternUrl = repositoryFileUrl("shared/ns-pattern.js"); const controlLibUrl = moduleDataUrl(` export const NS_RE = /^[a-z0-9-]+$/; export const WORKER_NAME_RE = /^[A-Za-z0-9][A-Za-z0-9_-]*$/; -export function configuredHostname(value) { return typeof value === "string" && value ? value : null; } +export { configuredHostname } from ${JSON.stringify(sharedNsPatternUrl)}; export function configuredPublicUrl() { return null; } export function platformVersionFromPackageJson() { return "wdl.test"; } export function projectAccessPrincipal(principal) { return principal || null; } @@ -172,6 +174,20 @@ test("control whoami uses sanitized forwarded proto for public URL hints", async }); }); +test("control whoami omits namespace URL when no public platform domain is configured", async () => { + const state = resetControlIndexState(); + state.authResult.principal = { kind: "ns", ns: "tenant-a" }; + + const response = await controlIndex.fetch( + new Request("http://control.example/whoami"), + {}, + /** @type {ExecutionContext} */ ({}) + ); + + const body = await readJsonResponse(response, 200); + assert.deepEqual(body.urls, { control: "http://control.example" }); +}); + test("control whoami ignores malformed forwarded proto values", async () => { resetControlIndexState(); diff --git a/tests/unit/control-lib.test.js b/tests/unit/control-lib.test.js index 945ec9d..2558b1b 100644 --- a/tests/unit/control-lib.test.js +++ b/tests/unit/control-lib.test.js @@ -4,6 +4,7 @@ import { loadControlLib } from "../helpers/load-control-lib.js"; import { readRepositoryJson } from "../helpers/load-shared-module.js"; import { RESERVED_NS } from "../../shared/ns-pattern.js"; import { + MIN_DYNAMIC_WORKER_COMPATIBILITY_DATE, WORKERD_EXPERIMENTAL_COMPAT_FLAGS, WORKERD_EXPERIMENTAL_COMPAT_FLAGS_SOURCE_VERSION, } from "../../shared/workerd-compat-flags.js"; @@ -27,8 +28,13 @@ const { NS_RE, parseControlRoute, configuredHostname, + platformDomainFromEnv, configuredPublicUrl, + parseWorkerdDependencyVersion, platformVersionFromPackageJson, + BundleMetaError, + parseBundleMeta, + bundleAssetPrefix, projectAccessPrincipal, parseR2DispatchPath, isAdminAcceptableNs, @@ -97,6 +103,110 @@ function deletePlatformTierFixture(ns) { RESERVED_NS.delete(ns); } +test("parseBundleMeta returns JSON objects", () => { + assert.deepEqual( + parseBundleMeta('{"routes":[],"vars":{"MODE":"test"}}', { + ns: "demo", + worker: "api", + version: "v3", + }), + { routes: [], vars: { MODE: "test" } } + ); +}); + +test("parseBundleMeta rejects missing, malformed, and non-object metadata with context", () => { + for (const raw of [undefined, null, "", "{bad", "null", "[]", "1", '"text"']) { + assert.throws( + () => parseBundleMeta(raw, { ns: "demo", worker: "api", version: "v3" }), + (err) => { + assert.ok(err instanceof BundleMetaError, `expected BundleMetaError for ${String(raw)}`); + const bundleError = /** @type {Error & { status: number, code: string, details: Record, cause: unknown }} */ (err); + assert.equal(bundleError.status, 500); + assert.equal(bundleError.code, "corrupt_meta"); + assert.equal(bundleError.message, "Corrupt __meta__ for demo/api/v3"); + assert.deepEqual(bundleError.details, { namespace: "demo", worker: "api", version: "v3" }); + assert.ok(bundleError.cause instanceof Error); + return true; + } + ); + } +}); + +test("bundleAssetPrefix reads only string prefixes from object-shaped assets", () => { + assert.equal(bundleAssetPrefix({ assets: { prefix: "assets/demo/" } }), "assets/demo/"); + assert.equal(bundleAssetPrefix({ assets: { prefix: "" } }), ""); + for (const assets of [null, [], "assets/demo/", 1, { prefix: 1 }]) { + assert.equal(bundleAssetPrefix({ assets }), null); + } +}); + +test("parseBundleMeta delegates error construction without changing parse context", () => { + const expected = new Error("routing-owned error"); + /** @type {{ namespace: string, worker: string, version: string, message: string, reason: string, cause: unknown } | null} */ + let failure = null; + assert.throws( + () => parseBundleMeta("[]", { + ns: "demo", + worker: "api", + version: "v3", + makeError(/** @type {{ namespace: string, worker: string, version: string, message: string, reason: string, cause: unknown }} */ value) { + failure = value; + return expected; + }, + }), + (err) => err === expected + ); + const captured = /** @type {{ namespace: string, worker: string, version: string, message: string, reason: string, cause: unknown } | null} */ ( + /** @type {unknown} */ (failure) + ); + assert.ok(captured); + assert.deepEqual( + { + namespace: captured.namespace, + worker: captured.worker, + version: captured.version, + message: captured.message, + reason: captured.reason, + }, + { + namespace: "demo", + worker: "api", + version: "v3", + message: "Corrupt __meta__ for demo/api/v3", + reason: "__meta__ must be a JSON object", + } + ); + assert.ok(captured.cause instanceof TypeError); +}); + +test("parseBundleMeta does not retain malformed persisted input in diagnostics", () => { + const marker = "SECRET_TOKEN_ABC"; + /** @type {{ reason: string, cause: unknown } | null} */ + let failure = null; + const expected = new Error("routing-owned error"); + + assert.throws( + () => parseBundleMeta(marker, { + ns: "demo", + worker: "api", + version: "v3", + makeError(/** @type {{ namespace: string, worker: string, version: string, message: string, reason: string, cause: unknown }} */ value) { + failure = value; + return expected; + }, + }), + (err) => err === expected + ); + + assert.ok(failure); + const captured = /** @type {{ reason: string, cause: unknown }} */ ( + /** @type {unknown} */ (failure) + ); + assert.equal(captured.reason, "__meta__ is not valid JSON"); + assert.ok(captured.cause instanceof SyntaxError); + assert.equal(JSON.stringify({ reason: captured.reason }).includes(marker), false); +}); + test("encodeReferrerMember: key order is canonical (alphabetical) regardless of caller shape", () => { const a = encodeReferrerMember({ binding: "AUTH", callerNs: "demo", callerWorker: "api", callerVersion: "v3", @@ -516,6 +626,17 @@ test("prepareBundle: durable object binding requires className-shaped class", () ); assert.equal(meta.bindings.ROOMS.type, "do"); assert.equal(meta.bindings.ROOMS.className, "Room"); + + const maxClassName = `R${"o".repeat(467)}`; + assert.doesNotThrow(() => prepareBundle("w.js", { "w.js": "x" }, { + bindings: { ROOMS: { type: "do", className: maxClassName } }, + })); + assert.throws( + () => prepareBundle("w.js", { "w.js": "x" }, { + bindings: { ROOMS: { type: "do", className: `${maxClassName}m` } }, + }), + /at most 468 bytes/ + ); }); test("prepareBundle: queue binding requires queue-name-shaped 'id'", () => { @@ -729,6 +850,42 @@ test("prepareBundle: experimental workerd compatibility flags are rejected", () ); }); +test("prepareBundle: error serialization flags match the effective compatibility date", () => { + assert.doesNotThrow(() => prepareBundle( + "w.js", + { "w.js": "x" }, + { + compatibilityDate: "2026-04-20", + compatibilityFlags: ["enhanced_error_serialization"], + } + )); + + /** @type {[string | undefined, string][]} */ + const cases = [ + ["2026-04-20", "legacy_error_serialization"], + ["2026-04-21", "legacy_error_serialization"], + ["2026-04-21", "enhanced_error_serialization"], + [undefined, "enhanced_error_serialization"], + ]; + for (const [compatibilityDate, flag] of cases) { + assert.throws( + () => prepareBundle( + "w.js", + { "w.js": "x" }, + { compatibilityDate, compatibilityFlags: [flag] } + ), + (err) => { + if (!(err instanceof Error)) return false; + const coded = /** @type {Error & { code?: unknown, status?: unknown }} */ (err); + return coded.code === "compatibility_flag_unsupported" && + coded.status === 400 && + coded.message.includes(flag); + }, + `${flag} should be rejected for ${compatibilityDate ?? "the default date"}` + ); + } +}); + test("prepareBundle: compatibilityDate validates shape before commit", () => { assert.equal( prepareBundle("w.js", { "w.js": "x" }, { compatibilityDate: "2026-04-24" }).meta.compatibilityDate, @@ -757,6 +914,14 @@ test("validateCompatibilityDate rejects future and unsupported workerd dates", ( validateCompatibilityDate("2026-06-20", new Date("2026-06-30T00:00:00Z")), "2026-06-20" ); + assert.equal( + validateCompatibilityDate(MIN_DYNAMIC_WORKER_COMPATIBILITY_DATE, new Date("2026-06-30T00:00:00Z")), + MIN_DYNAMIC_WORKER_COMPATIBILITY_DATE + ); + assert.throws( + () => validateCompatibilityDate("2026-03-31", new Date("2026-06-30T00:00:00Z")), + /older than WDL supports/ + ); assert.throws( () => validateCompatibilityDate("2026-06-15", new Date("2026-06-14T00:00:00Z")), /must not be later than today UTC/ @@ -768,15 +933,12 @@ test("validateCompatibilityDate rejects future and unsupported workerd dates", ( }); test("MAX_WORKER_COMPATIBILITY_DATE matches pinned workerd release plus seven days", () => { - assert.match( - PACKAGE_WORKERD_DEP, - /^[~^]?1\.\d{8}\.\d+$/, - `unexpected workerd dependency format ${PACKAGE_WORKERD_DEP}` - ); - assert.equal(PACKAGE_WORKERD_DEP.replace(/^[~^]/, ""), WORKERD_VERSION); - const match = /^1\.(\d{4})(\d{2})(\d{2})\.\d+$/.exec(WORKERD_VERSION); - assert.ok(match, `unexpected workerd version ${WORKERD_VERSION}`); - const max = new Date(Date.UTC(Number(match[1]), Number(match[2]) - 1, Number(match[3]) + 7)); + const parsed = parseWorkerdDependencyVersion(JSON.stringify({ + dependencies: { workerd: PACKAGE_WORKERD_DEP }, + })); + assert.ok(parsed, `unexpected workerd dependency format ${PACKAGE_WORKERD_DEP}`); + assert.equal(parsed.version, WORKERD_VERSION); + const max = new Date(Date.UTC(parsed.year, parsed.month - 1, parsed.day + 7)); const expected = [ String(max.getUTCFullYear()).padStart(4, "0"), String(max.getUTCMonth() + 1).padStart(2, "0"), @@ -854,6 +1016,13 @@ test("validateSecretKey: env-var grammar only", () => { assert.throws(() => validateSecretKey(bad), `"${bad}" should be rejected`); } assert.throws(() => validateSecretKey("__WDL_DO_BACKEND__"), /runtime-internal bindings/); + for (const reserved of ["__proto__", "constructor", "prototype", "toString", "valueOf"]) { + assert.throws( + () => validateSecretKey(reserved), + /reserved Object\.prototype key/, + `"${reserved}" should be rejected before persistence` + ); + } assert.throws(() => validateSecretKey("A".repeat(129)), /too long/); }); @@ -1962,6 +2131,10 @@ test("projectAccessPrincipal returns only the public principal shape", () => { test("configuredHostname accepts plain hosts and rejects URL injection shapes", () => { assert.equal(configuredHostname(" Workers.Local. "), "workers.local"); assert.equal(configuredHostname("workers.local."), "workers.local"); + assert.equal( + configuredHostname(`${"a".repeat(63)}.${"b".repeat(58)}.com`), + `${"a".repeat(63)}.${"b".repeat(58)}.com` + ); for (const bad of [ "", " ", @@ -1969,12 +2142,35 @@ test("configuredHostname accepts plain hosts and rejects URL injection shapes", "workers.local/path", "workers.local:8080", "workers local", + "workers.example#oops", + "workers.example?query", + "workers@example", + "workers..example", + ".workers.example", + "workers.example..", + "-workers.example", + "workers-.example", + "wörkers.example", + "K.example", + "workers.123", + `${"a".repeat(64)}.example`, + `${"a".repeat(63)}.${"b".repeat(59)}.com`, + `${"a".repeat(250)}.com`, null, ]) { assert.equal(configuredHostname(bad), null, `expected ${JSON.stringify(bad)} rejected`); } }); +test("platformDomainFromEnv normalizes configured values and owns the default", () => { + assert.equal(platformDomainFromEnv({}), "workers.local"); + assert.equal(platformDomainFromEnv({ PLATFORM_DOMAIN: " Workers.Example. " }), "workers.example"); + assert.throws( + () => platformDomainFromEnv({ PLATFORM_DOMAIN: "workers.example:8443" }), + /ALB-compatible ASCII DNS hostname/ + ); +}); + test("configuredPublicUrl returns a safe absolute base URL hint", () => { assert.equal(configuredPublicUrl(" https://assets.example/base/?token=secret#frag "), "https://assets.example/base"); assert.equal(configuredPublicUrl("http://assets.example/"), "http://assets.example"); @@ -2011,6 +2207,21 @@ test("platformVersionFromPackageJson derives WDL version from bundled workerd de ); }); +test("parseWorkerdDependencyVersion owns the bundled dependency grammar", () => { + assert.deepEqual( + parseWorkerdDependencyVersion(JSON.stringify({ dependencies: { workerd: "^1.20260531.12" } })), + { version: "1.20260531.12", year: 2026, month: 5, day: 31, patch: 12 } + ); + for (const source of [ + "{", + JSON.stringify({ dependencies: {} }), + JSON.stringify({ dependencies: { workerd: "2.20260531.1" } }), + JSON.stringify({ dependencies: { workerd: "1.20260531" } }), + ]) { + assert.equal(parseWorkerdDependencyVersion(source), null); + } +}); + test("normalizeWorkflows: validates deploy-wire workflow declarations", () => { assert.deepEqual(normalizeWorkflows(undefined), []); assert.deepEqual(normalizeWorkflows([ @@ -2214,6 +2425,20 @@ test("linkServiceBinding: target meta lookup throws → 502", async () => { 502, "failed to read target meta: redis blew up", "service_binding_target_meta_unavailable"); }); +test("linkServiceBinding: preserves target metadata domain errors", async () => { + const corruptMeta = new LinkError(500, "corrupt_meta", "Corrupt __meta__ for other/t/v1"); + await expectLinkError( + () => linkServiceBinding({ + callerNs: "caller", callerName: "c", bindingName: "T", + spec: { type: "service", service: "t", ns: "other" }, + ...makeLookups({ + versions: { "other/t": "v1" }, + metaThrows: corruptMeta, + }), + }), + 500, "Corrupt __meta__ for other/t/v1", "corrupt_meta"); +}); + test("linkServiceBinding: rejects runtime-reserved entrypoint at link time (defense in depth)", async () => { // Even if a manually-crafted binding makes it past validateBindings // (e.g. direct Redis writes during ops recovery), the linker is the diff --git a/tests/unit/control-lifecycle-indexes.test.js b/tests/unit/control-lifecycle-indexes.test.js index 2282322..0a57c4b 100644 --- a/tests/unit/control-lifecycle-indexes.test.js +++ b/tests/unit/control-lifecycle-indexes.test.js @@ -1,12 +1,21 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { importRepositoryModule } from "../helpers/load-shared-module.js"; +import { importRepositoryModule, readRepositoryJson } from "../helpers/load-shared-module.js"; + +const SCHEDULER_PROJECTION_CONTRACT = readRepositoryJson( + "tests/fixtures/scheduler-projection-contract.json" +); const { + cronEntryJson, + cronMetaJson, cronRefMember, + cronSlotExpireAt, cronSlotKey, cronWorkerKey, + queueConsumerFields, stageCronSlotRef, + stageCronProjection, stageCronWorkerIndexed, stageCronWorkerRemoved, stageD1ReferrerAdds, @@ -41,6 +50,11 @@ function recordMulti() { return this; }, /** @param {unknown[]} args */ + hDel(...args) { + calls.push(["hDel", ...args]); + return this; + }, + /** @param {unknown[]} args */ expireAt(...args) { calls.push(["expireAt", ...args]); return this; @@ -71,52 +85,79 @@ test("cron worker index helpers maintain non-authoritative discovery members", ( }); test("cron key helpers compose worker, slot, and ref members", () => { - assert.equal(cronWorkerKey("tenant", "worker"), "crons:tenant:worker"); - assert.equal(cronSlotKey(1_778_856_600_000), "cron-slot:1778856600000"); - assert.equal(cronRefMember("tenant", "worker", "cron-1", 7), "tenant:worker:cron-1:7"); + const cron = SCHEDULER_PROJECTION_CONTRACT.cron; + assert.equal(cronWorkerKey(cron.ns, cron.worker), cron.workerKey); + assert.equal(cronSlotKey(cron.slotMs), cron.slotKey); + assert.equal(cronSlotExpireAt(cron.slotMs), cron.slotExpireAt); + assert.equal(cronRefMember(cron.ns, cron.worker, cron.cronId, cron.gen), cron.reference); }); test("stageCronSlotRef writes ref and expiry together", () => { + const cron = SCHEDULER_PROJECTION_CONTRACT.cron; const multi = recordMulti(); - stageCronSlotRef(multi, "tenant", "worker", { - id: "cron-1", - gen: 7, - slot: 1_778_856_600_000, + stageCronSlotRef(multi, cron.ns, cron.worker, { + id: cron.cronId, + gen: cron.gen, + slot: cron.slotMs, }); assert.deepEqual(multi.calls, [ - ["sAdd", "cron-slot:1778856600000", "tenant:worker:cron-1:7"], - ["expireAt", "cron-slot:1778856600000", 1_778_857_200], + ["sAdd", cron.slotKey, cron.reference], + ["expireAt", cron.slotKey, cron.slotExpireAt], ]); }); -test("stageQueueConsumerProjection writes full optional queue consumer fields", () => { +test("stageCronProjection writes the shared scheduler projection contract", () => { + const cron = SCHEDULER_PROJECTION_CONTRACT.cron; const multi = recordMulti(); - stageQueueConsumerProjection(multi, "tenant", "worker", "v3", { - queue: "jobs", - maxBatchSize: 10, - maxBatchTimeoutMs: 250, - maxRetries: 4, - deadLetterQueue: "jobs-dlq", - retryDelaySeconds: 0, + const entry = { + id: cron.cronId, + cron: cron.entry.cron, + timezone: cron.entry.timezone, + gen: cron.entry.gen, + slot: cron.slotMs, + }; + stageCronProjection(multi, { + ns: cron.ns, + worker: cron.worker, + version: cron.meta.version, + cronKey: cron.workerKey, + existingHash: {}, + crons: [{ cron: entry.cron, timezone: entry.timezone }], + plan: { cronSeq: cron.meta.seq, addedWithPlacement: [entry], removed: [] }, }); + assert.equal(cronMetaJson(cron.meta.version, cron.meta.seq), cron.meta.json); + assert.equal(cronEntryJson(entry), cron.entry.json); + assert.deepEqual(multi.calls, [ + ["sAdd", cron.workerIndexKey, cron.workerKey], + ["hSet", cron.workerKey, "__meta__", cron.meta.json], + ["hSet", cron.workerKey, cron.cronId, cron.entry.json], + ["sAdd", cron.slotKey, cron.reference], + ["expireAt", cron.slotKey, cron.slotExpireAt], + ]); +}); + +test("stageQueueConsumerProjection writes full optional queue consumer fields", () => { + const contract = SCHEDULER_PROJECTION_CONTRACT.queueConsumer; + const multi = recordMulti(); + assert.equal(contract.input.queue, contract.queue); + assert.deepEqual( + queueConsumerFields(contract.worker, contract.version, contract.input), + contract.fields + ); + stageQueueConsumerProjection( + multi, + contract.ns, + contract.worker, + contract.version, + contract.input + ); + assert.deepEqual(multi.calls, [ - ["del", "queue-consumer:tenant:jobs"], - [ - "hSet", - "queue-consumer:tenant:jobs", - { - worker: "worker", - version: "v3", - max_batch_size: "10", - max_batch_timeout_ms: "250", - max_retries: "4", - dead_letter_queue: "jobs-dlq", - retry_delay_secs: "0", - }, - ], - ["sAdd", "queue-consumer-index", "queue-consumer:tenant:jobs"], + ["del", `queue-consumer:${contract.ns}:${contract.input.queue}`], + ["hSet", `queue-consumer:${contract.ns}:${contract.input.queue}`, contract.fields], + ["sAdd", "queue-consumer-index", `queue-consumer:${contract.ns}:${contract.input.queue}`], ]); }); diff --git a/tests/unit/control-logs-tail.test.js b/tests/unit/control-logs-tail.test.js index 2c1818c..b110279 100644 --- a/tests/unit/control-logs-tail.test.js +++ b/tests/unit/control-logs-tail.test.js @@ -1,8 +1,15 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { importRepositoryModuleFresh } from "../helpers/load-shared-module.js"; +import { + importRepositoryModuleFresh, + repositoryFileUrl, +} from "../helpers/load-shared-module.js"; +import { compileSharedAuthRoles } from "../helpers/load-auth-roles.js"; import { delay, waitUntil } from "../helpers/timing.js"; +const { sharedAuthRolesUrl } = await compileSharedAuthRoles(); +const SHARED_NS_PATTERN_URL = repositoryFileUrl("shared/ns-pattern.js"); + /** @param {{ keepaliveMs?: number }} [options] */ function loadLogsTailHandler(options = {}) { /** @type {Array<[RegExp | string, string]>} */ @@ -44,12 +51,12 @@ function loadLogsTailHandler(options = {}) { const redisDbFromEnv = (env, name) => Number(env?.[name] || 0);`, ], [ - /import \{ PLATFORM_TIER_RESERVED_NS \} from "shared-auth-roles";/, - "const PLATFORM_TIER_RESERVED_NS = new Set(['__platform__']);", + /from "shared-auth-roles";/, + `from ${JSON.stringify(sharedAuthRolesUrl)};`, ], [ - /import \{ isReservedNs \} from "shared-ns-pattern";/, - "const isReservedNs = (ns) => ns === '__system__' || ns === '__platform__';", + /from "shared-ns-pattern";/, + `from ${JSON.stringify(SHARED_NS_PATTERN_URL)};`, ], [ /import \{\n {2}WORKER_NAME_RE,\n {2}compareStreamIds,\n {2}isValidResumeId,\n {2}isValidWorkerName,\n\} from "control-lib";/, diff --git a/tests/unit/control-r2.test.js b/tests/unit/control-r2.test.js index c5bcf3f..96994d0 100644 --- a/tests/unit/control-r2.test.js +++ b/tests/unit/control-r2.test.js @@ -15,6 +15,9 @@ export class SigV4Client { `); const { + deleteR2Object, + getR2Object, + headR2Object, listR2Buckets, listR2Objects, makeR2AdminClient, @@ -22,6 +25,7 @@ const { "@wdl-dev/aws-sigv4": AWS_SIGV4_STUB_URL, "runtime-r2-utils": repositoryFileUrl("runtime/r2-utils.js"), "shared-s3-xml": repositoryFileUrl("shared/s3-xml.js"), + "shared-s3-retry": repositoryFileUrl("shared/s3-retry.js"), "shared-respond": repositoryFileUrl("shared/respond.js"), })); @@ -114,3 +118,41 @@ test("control R2 admin client restores S3 transient retry budget", () => { assert.ok(r2); assert.equal(r2.client.config.retries, 10); }); + +test("control R2 object methods share request, error, and not-found handling", async () => { + const getMock = r2AdminMock(new Response("missing", { status: 404 })); + assert.equal(await getR2Object({ + r2: getMock.r2, + ns: "demo", + bucketName: "uploads", + key: "a.txt", + requestId: "rid-get", + }), null); + assert.equal(getMock.calls[0].init?.method, "GET"); + assert.equal(new Headers(getMock.calls[0].init?.headers).get("x-request-id"), "rid-get"); + + const headMock = r2AdminMock(new Response("backend detail", { status: 503 })); + await assert.rejects( + () => headR2Object({ + r2: headMock.r2, + ns: "demo", + bucketName: "uploads", + key: "a.txt", + }), + /R2 admin HEAD failed with 503: backend detail/ + ); + + const deleteMock = r2AdminMock(new Response("missing", { status: 404 })); + assert.deepEqual(await deleteR2Object({ + r2: deleteMock.r2, + ns: "demo", + bucketName: "uploads", + key: "a.txt", + }), { + namespace: "demo", + bucket: "uploads", + key: "a.txt", + status: "ok", + }); + assert.equal(deleteMock.calls[0].init?.method, "DELETE"); +}); diff --git a/tests/unit/control-routing.test.js b/tests/unit/control-routing.test.js index bc2a7a3..7c94e9e 100644 --- a/tests/unit/control-routing.test.js +++ b/tests/unit/control-routing.test.js @@ -116,6 +116,42 @@ test("promoteWithRoutes removes queue consumer discovery index entries for remov )); }); +for (const [label, rawMeta] of [ + ["missing", null], + ["empty", ""], +]) { + test(`promoteWithRoutes rejects ${label} active bundle metadata before changing projections`, async () => { + const redis = makeRedis(); + seedBundle(redis, "v2", { routes: [], queueConsumers: [] }); + if (rawMeta !== null) { + redis.state.hashes.set(productionBundleKey("demo", "worker", "v1"), { + __meta__: rawMeta, + }); + } + redis.state.hashes.set("routes:demo", { worker: "v1" }); + redis.state.hashes.set("patterns:api.example", { + "/": patternProjection("demo", "worker", "v1", "exact", "/"), + }); + redis.state.hashes.set("queue-consumer:demo:jobs", { + worker: "worker", + version: "v1", + }); + + await assert.rejects( + promoteWithRoutes(redis, "demo", "worker", "v2"), + (err) => { + assertRoutingErrorShape(err, 500, "corrupt_meta"); + return true; + } + ); + + assert.equal(redis.state.hashes.get("routes:demo")?.worker, "v1"); + assert.equal(redis.state.hashes.get("queue-consumer:demo:jobs")?.version, "v1"); + assert.equal(redis.state.hashes.get("patterns:api.example")?.["/"], + patternProjection("demo", "worker", "v1", "exact", "/")); + }); +} + test("promoteWithRoutes skips empty platform route versions while checking exported as names", async () => { const redis = makeRedis(); redis.state.hashes.set(productionBundleKey("__platform__", "api", "v1"), { @@ -135,6 +171,131 @@ test("promoteWithRoutes skips empty platform route versions while checking expor ); }); +test("promoteWithRoutes rejects a platform route whose active metadata is missing", async () => { + const redis = makeRedis(); + redis.state.hashes.set(productionBundleKey("__platform__", "api", "v1"), { + __meta__: JSON.stringify({ exports: [{ entrypoint: "default", as: "demo" }] }), + }); + redis.state.hashes.set("routes:__platform__", { other: "v2" }); + + await assert.rejects( + promoteWithRoutes(redis, "__platform__", "api", "v1"), + (err) => { + assertRoutingErrorShape(err, 500, "corrupt_meta"); + return true; + } + ); + + assert.equal(redis.state.hashes.get("routes:__platform__")?.api, undefined); +}); + +test("promoteWithRoutes retries from the new active when the prior bundle is deleted", async () => { + const redis = makeRedis(); + seedBundle(redis, "v1", { routes: [], queueConsumers: [] }); + seedBundle(redis, "v2", { routes: [], queueConsumers: [] }); + seedBundle(redis, "v3", { routes: [], queueConsumers: [consumerWithoutOptions] }); + redis.state.hashes.set("routes:demo", { worker: "v1" }); + + const originalSession = redis.session.bind(redis); + let attempts = 0; + redis.session = async (fn) => { + attempts += 1; + return await originalSession(async (iso) => { + if (attempts !== 1) return await fn(iso); + const originalHGet = iso.hGet.bind(iso); + let raced = false; + return await fn({ + ...iso, + async hGet(key, field) { + const value = await originalHGet(key, field); + if (!raced && key === "routes:demo" && field === "worker") { + raced = true; + redis.state.hashes.set("routes:demo", { worker: "v3" }); + redis.state.hashes.set("queue-consumer:demo:jobs", { + worker: "worker", + version: "v3", + max_batch_size: "5", + max_batch_timeout_ms: "2000", + max_retries: "3", + }); + redis.state.sets.set( + "queue:index:consumers", + new Set(["queue-consumer:demo:jobs"]) + ); + redis.state.hashes.delete(productionBundleKey("demo", "worker", "v1")); + } + return value; + }, + }); + }); + }; + + await promoteWithRoutes(redis, "demo", "worker", "v2"); + + assert.equal(attempts, 2); + assert.equal(redis.state.hashes.get("routes:demo")?.worker, "v2"); + assert.equal(redis.state.hashes.has("queue-consumer:demo:jobs"), false); + assert.equal( + redis.state.sets.get("queue:index:consumers")?.has("queue-consumer:demo:jobs") ?? false, + false + ); +}); + +test("promoteWithRoutes retries when a platform route is deleted during as validation", async () => { + const redis = makeRedis(); + redis.state.hashes.set(productionBundleKey("__platform__", "api", "v1"), { + __meta__: JSON.stringify({ exports: [{ entrypoint: "default", as: "demo" }] }), + }); + redis.state.hashes.set(productionBundleKey("__platform__", "other", "v2"), { + __meta__: JSON.stringify({ exports: [{ entrypoint: "default", as: "other" }] }), + }); + redis.state.hashes.set("routes:__platform__", { other: "v2" }); + + const originalSession = redis.session.bind(redis); + let attempts = 0; + redis.session = async (fn) => { + attempts += 1; + return await originalSession(async (iso) => { + if (attempts !== 1) return await fn(iso); + const originalHGetMany = iso.hGetMany.bind(iso); + let raced = false; + return await fn({ + ...iso, + async hGetMany(pairs) { + if (!raced && pairs.some(([key]) => + key === productionBundleKey("__platform__", "other", "v2") + )) { + raced = true; + delete redis.state.hashes.get("routes:__platform__")?.other; + redis.state.hashes.delete(productionBundleKey("__platform__", "other", "v2")); + } + return await originalHGetMany(pairs); + }, + }); + }); + }; + + await promoteWithRoutes(redis, "__platform__", "api", "v1"); + + assert.equal(attempts, 2); + assert.equal(redis.state.hashes.get("routes:__platform__")?.api, "v1"); +}); + +test("promoteWithRoutes rejects non-object candidate bundle metadata", async () => { + const redis = makeRedis(); + redis.state.hashes.set(productionBundleKey("demo", "worker", "v1"), { + __meta__: "[]", + }); + + await assert.rejects( + promoteWithRoutes(redis, "demo", "worker", "v1"), + (err) => { + assertRoutingErrorShape(err, 500, "corrupt_meta"); + return true; + } + ); +}); + test("promoteWithRoutes rejects malformed cron metadata", async () => { const redis = makeRedis(); /** @type {Array<{ level: string, event: string, fields: any }>} */ @@ -223,6 +384,31 @@ test("promoteWithRoutes watches and rejects missing service-binding target bundl assert.equal(redis.state.hashes.has(productionBundleKey("demo", "worker", "v2")), false); }); +test("promoteWithRoutes rejects empty service-binding target metadata", async () => { + const redis = makeRedis(); + seedBundle(redis, "v1", { + bindings: { + TARGET: { + type: "service", + ns: "other", + service: "api", + version: "v3", + }, + }, + }); + redis.state.hashes.set(productionBundleKey("other", "api", "v3"), { __meta__: "" }); + + await assert.rejects( + promoteWithRoutes(redis, "demo", "worker", "v1"), + (err) => { + const shaped = assertRoutingErrorShape(err, 409, "service_binding_dependency_missing"); + assert.equal(shaped.details.broken_dependency.targetNs, "other"); + return true; + } + ); + assert.equal(redis.state.hashes.get("routes:demo")?.worker, undefined); +}); + test("promoteWithRoutes batches service-binding dependency watches", async () => { const redis = makeRedis(); seedBundle(redis, "v1", { @@ -280,6 +466,77 @@ test("bumpActiveAndPromote also rewrites full queue consumer projection", async }); }); +test("bumpActiveAndPromote batches pattern projection reads across hosts", async () => { + const redis = makeRedis(); + const routes = ["api.example", "admin.example"].map((host) => ({ + host, + slot: "/*", + kind: "prefix", + value: "/", + })); + seedBundle(redis, "v1", { routes }); + redis.state.hashes.set("routes:demo", { worker: "v1" }); + redis.state.sets.set("hosts:demo", new Set(routes.map((route) => route.host))); + for (const route of routes) { + redis.state.hashes.set(`patterns:${route.host}`, { + [route.slot]: patternProjection("demo", "worker", "v1", route.kind, route.value), + }); + } + redis.state.strings.set("worker:demo:worker:next_version", "1"); + + await bumpActiveAndPromote(redis, "demo", "worker"); + + assert.deepEqual( + redis.state.commands.filter(([command]) => command === "hGetAllMany"), + [["hGetAllMany", ["patterns:api.example", "patterns:admin.example"]]] + ); + assert.equal( + redis.state.commands.some(([command, key]) => + command === "hGetAll" && typeof key === "string" && key.startsWith("patterns:") + ), + false + ); +}); + +test("bumpActiveAndPromote retries when active changes before source metadata read", async () => { + const redis = makeRedis(); + seedBundle(redis, "v1", {}); + seedBundle(redis, "v3", {}); + redis.state.hashes.set("routes:demo", { worker: "v1" }); + redis.state.strings.set("worker:demo:worker:next_version", "3"); + + const originalSession = redis.session.bind(redis); + let attempts = 0; + redis.session = async (fn) => { + attempts += 1; + return await originalSession(async (iso) => { + if (attempts !== 1) return await fn(iso); + const originalHGet = iso.hGet.bind(iso); + let raced = false; + return await fn({ + ...iso, + async hGet(key, field) { + const value = await originalHGet(key, field); + if (!raced && key === "routes:demo" && field === "worker") { + raced = true; + redis.state.hashes.set("routes:demo", { worker: "v3" }); + redis.state.hashes.delete(productionBundleKey("demo", "worker", "v1")); + } + return value; + }, + }); + }); + }; + + const result = await bumpActiveAndPromote(redis, "demo", "worker"); + + assert.equal(attempts, 2); + assert.equal(result.previousVersion, "v3"); + assert.equal(result.version, "v4"); + assert.equal(redis.state.hashes.get("routes:demo")?.worker, "v4"); + assert.ok(redis.state.hashes.has(productionBundleKey("demo", "worker", "v4"))); +}); + test("promoteWithRoutes rejects missing D1 dependency databases", async () => { const redis = makeRedis(); seedBundle(redis, "v1", { @@ -381,6 +638,21 @@ test("bumpActiveAndPromote aborts copy and route flip when staged writes fail", assert.ok(redis.state.watched.includes("secrets:demo:worker")); }); +test("bumpActiveAndPromote preserves bundle_copy_failed for a missing source bundle", async () => { + const redis = makeRedis(); + redis.state.hashes.set("routes:demo", { worker: "v1" }); + redis.state.strings.set("worker:demo:worker:next_version", "1"); + + await assert.rejects( + bumpActiveAndPromote(redis, "demo", "worker"), + (err) => { + assertRoutingErrorShape(err, 500, "bundle_copy_failed"); + return true; + } + ); + assert.equal(redis.state.hashes.has(productionBundleKey("demo", "worker", "v2")), false); +}); + test("bumpActiveAndPromote rejects active routes that no longer declare their hosts", async () => { const redis = makeRedis(); seedBundle(redis, "v1", { @@ -404,6 +676,40 @@ test("bumpActiveAndPromote rejects active routes that no longer declare their ho ); }); +/** @type {Array<[string, string, number, string]>} */ +const bumpProjectionFailureCases = [ + ["malformed", "not-a-projection", 500, "corrupt_pattern_projection"], + ["foreign", patternProjection("other", "site", "v7", "prefix", "/"), 409, "route_conflict"], +]; +for (const [label, held, expectedStatus, expectedCode] of bumpProjectionFailureCases) { + test(`bumpActiveAndPromote fails closed on ${label} held pattern projections`, async () => { + const redis = makeRedis(); + seedBundle(redis, "v1", { + routes: [{ + host: "app.workers.example", + slot: "/*", + kind: "prefix", + value: "/", + }], + }); + redis.state.hashes.set("routes:demo", { worker: "v1" }); + redis.state.hashes.set("patterns:app.workers.example", { "/*": held }); + redis.state.sets.set("hosts:demo", new Set(["app.workers.example"])); + redis.state.strings.set("worker:demo:worker:next_version", "1"); + + await assert.rejects( + bumpActiveAndPromote(redis, "demo", "worker"), + (err) => { + assertRoutingErrorShape(err, expectedStatus, expectedCode); + return true; + } + ); + + assert.equal(redis.state.hashes.get("routes:demo")?.worker, "v1"); + assert.equal(redis.state.hashes.has(productionBundleKey("demo", "worker", "v2")), false); + }); +} + test("bumpActiveAndPromote rejects malformed cron metadata", async () => { const redis = makeRedis(); /** @type {Array<{ level: string, event: string, fields: any }>} */ @@ -522,6 +828,30 @@ test("promoteWithRoutes rejects a custom host already owned by another namespace assert.equal(redis.state.hashes.get("routes:demo")?.worker, undefined); }); +test("promoteWithRoutes fails closed on malformed occupied pattern projections", async () => { + const redis = makeRedis(); + seedBundle(redis, "v1", { + routes: [{ + host: "app.workers.example", + slot: "/*", + kind: "prefix", + value: "/", + }], + }); + redis.state.sets.set("hosts:demo", new Set(["app.workers.example"])); + redis.state.hashes.set("patterns:app.workers.example", { "/*": "not-a-projection" }); + + await assert.rejects( + promoteWithRoutes(redis, "demo", "worker", "v1"), + (err) => { + const shaped = assertRoutingErrorShape(err, 500, "corrupt_pattern_projection"); + assert.deepEqual(shaped.details, { host: "app.workers.example", slot: "/*" }); + return true; + } + ); + assert.equal(redis.state.hashes.get("routes:demo")?.worker, undefined); +}); + test("reconcileHosts stages declared host indexes and pattern invalidation", async () => { const redis = makeRedis(); diff --git a/tests/unit/control-s3.test.js b/tests/unit/control-s3.test.js index db7caec..bd79be6 100644 --- a/tests/unit/control-s3.test.js +++ b/tests/unit/control-s3.test.js @@ -9,6 +9,7 @@ import { const { makeS3Client } = await importRepositoryModule("control/s3.js", [ [/import \{ SigV4Client \} from "@wdl-dev\/aws-sigv4";/, "class SigV4Client { constructor(options) { this.options = options; } }"], [/from "runtime-r2-utils";/g, `from ${JSON.stringify(repositoryFileUrl("runtime/r2-utils.js"))};`], + [/from "shared-s3-retry";/g, `from ${JSON.stringify(repositoryFileUrl("shared/s3-retry.js"))};`], ]); test("makeS3Client requires credentials outside explicit local/mock endpoints", () => { diff --git a/tests/unit/control-secret-envelope-handlers.test.js b/tests/unit/control-secret-envelope-handlers.test.js index 166710d..8a8dee3 100644 --- a/tests/unit/control-secret-envelope-handlers.test.js +++ b/tests/unit/control-secret-envelope-handlers.test.js @@ -11,15 +11,21 @@ import { moduleDataUrl, readRepositoryFile, repositoryFileUrl, + repositoryModuleDataUrl, } from "../helpers/load-shared-module.js"; -import { createFakeRedis } from "../helpers/mocks/fake-redis.js"; +import { createFakeRedis, sharedRedisStubUrl } from "../helpers/mocks/fake-redis.js"; import { CONTROL_ROUTING_TEST_URL } from "../helpers/load-control-routing.js"; +import { compileControlGraph } from "../helpers/load-control-lib.js"; import { readJsonResponse } from "../helpers/response-json.js"; const SECRET_ENVELOPE_URL = repositoryFileUrl("shared/secret-envelope.js"); -const SHARED_ERRORS_URL = repositoryFileUrl("shared/errors.js"); const SHARED_VERSION_URL = repositoryFileUrl("shared/version.js"); const SHARED_SECRET_KEYS_URL = repositoryFileUrl("shared/secret-keys.js"); +const RUNTIME_ENV_BUILD_URL = repositoryModuleDataUrl("runtime/load/env-build.js", [ + [/from "shared-ns-pattern";/, `from ${JSON.stringify(repositoryFileUrl("shared/ns-pattern.js"))};`], + [/from "shared-version";/, `from ${JSON.stringify(SHARED_VERSION_URL)};`], +]); +const { libUrl: PRODUCTION_CONTROL_LIB_URL } = await compileControlGraph(); const env = { SECRET_ENVELOPE_LOCAL_KEY_B64: "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=", SECRET_ENVELOPE_KID: "local:test:secret-envelope:v1", @@ -114,11 +120,13 @@ async function withNamespaceSecretRedis(state, setup, callback) { } const validateSecretKeyStubSource = ` +import { RESERVED_OBJECT_KEYS } from ${JSON.stringify(repositoryFileUrl("shared/ns-pattern.js"))}; const SECRET_KEY_RE = /^[A-Za-z_][A-Za-z0-9_]*$/; const WDL_RESERVED_BINDING_RE = /^__WDL_[A-Za-z0-9_]*__$/; export function validateSecretKey(key) { if (typeof key !== "string" || !SECRET_KEY_RE.test(key)) throw new Error("bad key"); if (WDL_RESERVED_BINDING_RE.test(key)) throw new Error("reserved key"); + if (RESERVED_OBJECT_KEYS.has(key)) throw new Error("reserved object key"); if (key.length > 128) throw new Error("key too long"); } `; @@ -137,17 +145,11 @@ function secretPutUrl(controlSharedUrl, controlLibUrl) { } function envBudgetUrl() { - const sharedRedisUrl = moduleDataUrl(` -export class WatchError extends Error { - constructor(message = "watched key changed") { - super(message); - this.name = "WatchError"; - } -} -`); + const sharedRedisUrl = sharedRedisStubUrl(); const source = applyModuleReplacements(readRepositoryFile("control/env-budget.js"), [ + [/from "control-lib";/, `from ${JSON.stringify(PRODUCTION_CONTROL_LIB_URL)};`], + [/from "runtime-load-env-build";/, `from ${JSON.stringify(RUNTIME_ENV_BUILD_URL)};`], [/from "shared-secret-envelope";/, `from ${JSON.stringify(SECRET_ENVELOPE_URL)};`], - [/from "shared-errors";/, `from ${JSON.stringify(SHARED_ERRORS_URL)};`], [/from "shared-version";/, `from ${JSON.stringify(SHARED_VERSION_URL)};`], [/from "shared-redis";/, `from ${JSON.stringify(sharedRedisUrl)};`], ]); @@ -271,6 +273,44 @@ test("namespace secret PUT stores an envelope instead of plaintext", async () => }); }); +test("namespace secret PUT hides envelope configuration details and logs them", async () => { + const logStart = namespaceSecretState.logs.length; + await withNamespaceSecretRedis(namespaceSecretState, () => {}, async (redis) => { + const response = await handle({ + request: new Request("http://control.test/ns/demo/secrets/TOKEN", { + method: "PUT", + body: JSON.stringify({ value: "plain-secret" }), + }), + env: {}, + method: "PUT", + nsName: "demo", + secretKey: "TOKEN", + requestId: "rid-secret-unconfigured", + }); + + const body = await readJsonResponse(response, 503); + assert.deepEqual(body, { + error: "secret_encryption_unconfigured", + message: "Internal error", + }); + assert.deepEqual(namespaceSecretState.logs.slice(logStart), [{ + level: "error", + event: "ns_secret_mutation_rejected", + fields: { + request_id: "rid-secret-unconfigured", + namespace: "demo", + key: "TOKEN", + method: "PUT", + status: 503, + reason: "secret_encryption_unconfigured", + error_message: "SECRET_ENVELOPE_KID must be a canonical local provider kid", + error_detail: "SECRET_ENVELOPE_KID must be a canonical local provider kid", + }, + }]); + assert.equal(redis.ops.length, 0); + }); +}); + test("namespace secret mutation rejects invalid keys through shared validator", async () => { await withNamespaceSecretRedis(namespaceSecretState, () => {}, async (redis) => { const response = await handle({ @@ -291,6 +331,26 @@ test("namespace secret mutation rejects invalid keys through shared validator", }); }); +test("namespace secret mutation rejects Object.prototype keys before persistence", async () => { + await withNamespaceSecretRedis(namespaceSecretState, () => {}, async (redis) => { + const response = await handle({ + request: new Request("http://control.test/ns/demo/secrets/constructor", { + method: "PUT", + body: JSON.stringify({ value: "plain-secret" }), + }), + env, + method: "PUT", + nsName: "demo", + secretKey: "constructor", + requestId: "rid-secret", + }); + + const body = await readJsonResponse(response, 400); + assert.equal(body.error, "invalid_request"); + assert.equal(redis.ops.length, 0); + }); +}); + test("namespace secret PUT accepts lowercase secret keys like production", async () => { await withNamespaceSecretRedis(namespaceSecretState, () => {}, async (redis) => { const response = await handle({ @@ -367,6 +427,51 @@ test("namespace secret PUT checks retained worker versions before storing", asyn }); }); +test("namespace secret PUT returns corrupt_meta for invalid retained bundle metadata", async () => { + const logStart = namespaceSecretState.logs.length; + await withNamespaceSecretRedis(namespaceSecretState, (redis) => { + redis.sets.set("workers:demo", new Set(["api"])); + seedWorkerSecretVersions(redis, ["v1"]); + redis.hashes.set("worker:demo:api:v:1", { __meta__: "[]" }); + }, async (redis) => { + const response = await handle({ + request: new Request("http://control.test/ns/demo/secrets/TOKEN", { + method: "PUT", + body: JSON.stringify({ value: "plain-secret" }), + }), + env, + method: "PUT", + nsName: "demo", + secretKey: "TOKEN", + requestId: "rid-ns-secret-corrupt-meta", + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_meta"); + assert.equal(body.namespace, "demo"); + assert.equal(body.worker, "api"); + assert.equal(body.version, "v1"); + assert.equal(body.detail, undefined); + assert.deepEqual(namespaceSecretState.logs.slice(logStart).find((entry) => + entry.event === "ns_secret_mutation_rejected" + ), { + level: "error", + event: "ns_secret_mutation_rejected", + fields: { + request_id: "rid-ns-secret-corrupt-meta", + namespace: "demo", + key: "TOKEN", + method: "PUT", + status: 500, + reason: "corrupt_meta", + error_message: "Corrupt __meta__ for demo/api/v1", + error_detail: "__meta__ must be a JSON object", + }, + }); + assert.deepEqual(redisHSetAttempts(redis, "secrets:demo"), []); + }); +}); + test("namespace secret PUT budgets active worker versions with their real active version", async () => { const nsSecrets = { TOKEN: "plain-secret" }; const padLength = WORKER_LOADER_ENV_MAX_BYTES - @@ -605,16 +710,25 @@ const workerSecretState = /** @type {ReturnType entry.event === "secret_mutation_rejected"); +} + const workerControlSharedUrl = controlSharedHarnessUrl(WORKER_SECRET_STATE_GLOBAL); const workerLibStubUrl = moduleDataUrl(` ${validateSecretKeyStubSource} +export { workflowDefsKey } from ${JSON.stringify(PRODUCTION_CONTROL_LIB_URL)}; export const deleteLockKey = (ns, worker) => \`worker-delete-lock:\${ns}:\${worker}\`; export const workerVersionsKey = (ns, worker) => \`worker-versions:\${ns}:\${worker}\`; export const routesKey = (ns) => \`routes:\${ns}\`; export const workersIndexKey = (ns) => \`workers:\${ns}\`; `); const lifecycleStubUrl = moduleDataUrl(` -export function stageWorkerHidden() {} +export function stageWorkerHidden(multi, ns, name) { + multi.sRem(\`workers:\${ns}\`, name); +} export function stageWorkerVisible(multi, ns, name) { multi.sAdd(\`workers:\${ns}\`, name); } @@ -633,6 +747,47 @@ const workerSrc = applyModuleReplacements(readRepositoryFile("control/handlers/w ]); const { handle: workerHandle } = await import(moduleDataUrl(workerSrc)); +test("worker secret PUT hides envelope configuration details and logs them", async () => { + const state = workerSecretState; + const logStart = state.logs.length; + await withWorkerSecretRedis(state, () => {}, async (redis) => { + const response = await workerHandle({ + request: new Request("http://control.test/ns/demo/workers/api/secrets/TOKEN", { + method: "PUT", + body: JSON.stringify({ value: "plain-secret" }), + }), + env: {}, + method: "PUT", + ns: "demo", + name: "api", + subPath: ["TOKEN"], + requestId: "rid-worker-secret-unconfigured", + }); + + const body = await readJsonResponse(response, 503); + assert.deepEqual(body, { + error: "secret_encryption_unconfigured", + message: "Internal error", + }); + assert.deepEqual(workerSecretRejectionLogsSince(logStart), [{ + level: "error", + event: "secret_mutation_rejected", + fields: { + request_id: "rid-worker-secret-unconfigured", + namespace: "demo", + worker: "api", + key: "TOKEN", + method: "PUT", + status: 503, + reason: "secret_encryption_unconfigured", + error_message: "SECRET_ENVELOPE_KID must be a canonical local provider kid", + error_detail: "SECRET_ENVELOPE_KID must be a canonical local provider kid", + }, + }]); + assert.equal(redis.ops.length, 0); + }); +}); + test("worker secret PUT encrypts before WATCH retries and reuses the envelope", async () => { const state = workerSecretState; await withWorkerSecretRedis(state, (redis) => { @@ -681,6 +836,77 @@ test("worker secret PUT encrypts before WATCH retries and reuses the envelope", }); }); +test("worker secret PUT returns corrupt_meta for invalid active bundle metadata", async () => { + const state = workerSecretState; + await withWorkerSecretRedis(state, (redis) => { + redis.hashes.set("worker:demo:api:v:1", { __meta__: "[]" }); + }, async (redis) => { + const response = await workerHandle({ + request: new Request("http://control.test/ns/demo/workers/api/secrets/TOKEN", { + method: "PUT", + body: JSON.stringify({ value: "plain-secret" }), + }), + env, + method: "PUT", + ns: "demo", + name: "api", + subPath: ["TOKEN"], + requestId: "rid-worker-secret-corrupt-meta", + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_meta"); + assert.equal(body.message, "Internal error"); + assert.equal(body.detail, undefined); + assert.deepEqual(redisHSetAttempts(redis, "secrets:demo:api"), []); + }); +}); + +test("worker secret PUT logs retained metadata diagnostics without exposing them", async () => { + const state = workerSecretState; + const logStart = state.logs.length; + await withWorkerSecretRedis(state, (redis) => { + redis.hashes.delete("routes:demo"); + seedWorkerSecretVersions(redis, ["v1"]); + redis.hashes.set("worker:demo:api:v:1", { __meta__: "[]" }); + }, async (redis) => { + const response = await workerHandle({ + request: new Request("http://control.test/ns/demo/workers/api/secrets/TOKEN", { + method: "PUT", + body: JSON.stringify({ value: "plain-secret" }), + }), + env, + method: "PUT", + ns: "demo", + name: "api", + subPath: ["TOKEN"], + requestId: "rid-worker-secret-retained-corrupt-meta", + }); + + const body = await readJsonResponse(response, 500); + assert.equal(body.error, "corrupt_meta"); + assert.equal(body.detail, undefined); + assert.deepEqual(state.logs.slice(logStart).find((entry) => + entry.event === "secret_mutation_rejected" + ), { + level: "error", + event: "secret_mutation_rejected", + fields: { + request_id: "rid-worker-secret-retained-corrupt-meta", + namespace: "demo", + worker: "api", + key: "TOKEN", + method: "PUT", + status: 500, + reason: "corrupt_meta", + error_message: "Corrupt __meta__ for demo/api/v1", + error_detail: "__meta__ must be a JSON object", + }, + }); + assert.deepEqual(redisHSetAttempts(redis, "secrets:demo:api"), []); + }); +}); + test("worker secret mutation rejects invalid keys through shared validator", async () => { const state = workerSecretState; const opsBefore = state.redis.ops.length; @@ -735,8 +961,42 @@ test("worker secret PUT active precheck reads the shared fake Redis state", asyn }); }); +test("worker secret DELETE keeps a definitions-only worker discoverable", async () => { + const state = workerSecretState; + const encrypted = await encryptSecretValue("plain", { + env, + hashKey: "secrets:demo:api", + fieldName: "TOKEN", + }); + await withWorkerSecretRedis(state, (redis) => { + redis.hashes.delete("routes:demo"); + redis.hashes.set("secrets:demo:api", { TOKEN: encrypted }); + redis.hashes.set("wf:defs:demo:api", { flow: "{}" }); + redis.sets.set("workers:demo", new Set(["api"])); + }, async (redis) => { + const response = await workerHandle({ + request: new Request("http://control.test/ns/demo/workers/api/secrets/TOKEN", { + method: "DELETE", + }), + env, + method: "DELETE", + ns: "demo", + name: "api", + subPath: ["TOKEN"], + requestId: "rid-worker-secret-definitions-only", + }); + + const body = await readJsonResponse(response, 200); + assert.equal(body.deleted, true); + assert.equal(redis.hashes.has("secrets:demo:api"), false); + assert.equal(redis.sets.get("workers:demo")?.has("api"), true); + assert.equal(redis.watched.includes("wf:defs:demo:api"), true); + }); +}); + test("worker secret PUT maps active bump delete-lock errors to deleting", async () => { const state = workerSecretState; + const logStart = state.logs.length; await withWorkerSecretRedis(state, (redis) => { redis.strings.set("worker-delete-lock:demo:api", "holder-token"); }, async (redis) => { @@ -756,11 +1016,26 @@ test("worker secret PUT maps active bump delete-lock errors to deleting", async const body = await readJsonResponse(response, 409); assert.equal(body.error, "deleting"); assert.equal(redis.ops.length, 0); + assert.deepEqual(workerSecretRejectionLogsSince(logStart), [{ + level: "warn", + event: "secret_mutation_rejected", + fields: { + request_id: "rid-worker-secret-delete-lock", + namespace: "demo", + worker: "api", + key: "KEY", + method: "PUT", + status: 409, + reason: "deleting", + error_message: "deleting", + }, + }]); }); }); test("worker secret PUT maps active bump contention to secret mutation contention", async () => { const state = workerSecretState; + const logStart = state.logs.length; await withWorkerSecretRedis(state, (redis) => { redis.execFailures = 5; }, async (redis) => { @@ -780,6 +1055,57 @@ test("worker secret PUT maps active bump contention to secret mutation contentio const body = await readJsonResponse(response, 503); assert.equal(body.error, "secret_mutation_contention"); assert.equal(redis.ops.some((op) => op[0] === "hSet" && op[1] === "secrets:demo:api"), false); + assert.deepEqual(workerSecretRejectionLogsSince(logStart), [{ + level: "error", + event: "secret_mutation_rejected", + fields: { + request_id: "rid-worker-secret-bump-contention", + namespace: "demo", + worker: "api", + key: "KEY", + method: "PUT", + status: 503, + reason: "secret_mutation_contention", + error_message: "active version changed during secret mutation; retry later", + }, + }]); + }); +}); + +test("worker secret DELETE precheck logs direct mutation aborts through the shared rejection shape", async () => { + const state = workerSecretState; + const logStart = state.logs.length; + await withWorkerSecretRedis(state, (redis) => { + redis.strings.set("worker-delete-lock:demo:api", "holder-token"); + }, async () => { + const response = await workerHandle({ + request: new Request("http://control.test/ns/demo/workers/api/secrets/KEY", { + method: "DELETE", + }), + env, + method: "DELETE", + ns: "demo", + name: "api", + subPath: ["KEY"], + requestId: "rid-worker-secret-delete-precheck", + }); + + const body = await readJsonResponse(response, 409); + assert.equal(body.error, "deleting"); + assert.deepEqual(workerSecretRejectionLogsSince(logStart), [{ + level: "warn", + event: "secret_mutation_rejected", + fields: { + request_id: "rid-worker-secret-delete-precheck", + namespace: "demo", + worker: "api", + key: "KEY", + method: "DELETE", + status: 409, + reason: "deleting", + error_message: "deleting", + }, + }]); }); }); diff --git a/tests/unit/control-shared-stub.test.js b/tests/unit/control-shared-stub.test.js new file mode 100644 index 0000000..ead44cf --- /dev/null +++ b/tests/unit/control-shared-stub.test.js @@ -0,0 +1,63 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + jsonError as productionJsonError, + jsonResponse as productionJsonResponse, + sanitizeJsonErrorDetails as productionSanitizeJsonErrorDetails, +} from "../../shared/respond.js"; +import { controlSharedStubUrl } from "../helpers/control-shared-stub.js"; +import { readJsonResponse } from "../helpers/response-json.js"; + +const stub = await import(controlSharedStubUrl()); + +test("control shared stub re-exports the production JSON response contract", () => { + assert.equal(stub.jsonError, productionJsonError); + assert.equal(stub.jsonResponse, productionJsonResponse); + assert.equal(stub.sanitizeJsonErrorDetails, productionSanitizeJsonErrorDetails); +}); + +test("control shared stub enforces streamed request limits through shared bounded-body", async () => { + let canceled = false; + const requestInit = /** @type {RequestInit} */ (/** @type {unknown} */ ({ + method: "POST", + body: new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('{"a":')); + controller.enqueue(new TextEncoder().encode('"too large"}')); + }, + cancel() { + canceled = true; + }, + }), + duplex: "half", + })); + const request = new Request("http://control.test/", requestInit); + + const result = await stub.readJsonBody(request, { maxBytes: 8 }); + assert.ok("response" in result); + await readJsonResponse(result.response, 413); + assert.equal(canceled, true); +}); + +test("control shared stub matches production internal-auth header validation", async () => { + const missing = await import(controlSharedStubUrl("export const state = { env: {} };")); + assert.throws( + () => missing.controlInternalJsonHeaders(), + /WDL_INTERNAL_AUTH_TOKEN must be configured/, + ); + + const invalid = await import(controlSharedStubUrl(` + export const state = { env: { WDL_INTERNAL_AUTH_TOKEN: "bad token" } }; + `)); + assert.throws( + () => invalid.controlInternalJsonHeaders(), + /visible ASCII without whitespace or commas/, + ); + + const valid = await import(controlSharedStubUrl(` + export const state = { env: { WDL_INTERNAL_AUTH_TOKEN: "test-token" } }; + `)); + const headers = valid.controlInternalJsonHeaders(); + assert.equal(headers.get("content-type"), "application/json"); + assert.equal(headers.get("x-wdl-internal-auth"), "test-token"); +}); diff --git a/tests/unit/control-shared.test.js b/tests/unit/control-shared.test.js index 6f0fb43..a005ce3 100644 --- a/tests/unit/control-shared.test.js +++ b/tests/unit/control-shared.test.js @@ -12,6 +12,7 @@ import { readRepositoryFile, repositoryFileUrl, } from "../helpers/load-shared-module.js"; +import { sharedRedisStubUrl } from "../helpers/mocks/fake-redis.js"; import { installMockProperty } from "../helpers/mock-global.js"; import { parseJsonObjectRequestBody } from "../helpers/request-body.js"; import { assertJsonResponse } from "../helpers/response-json.js"; @@ -19,9 +20,8 @@ import { sharedInternalAuthUrl } from "../helpers/runtime-proxy-stub.js"; const TEST_INTERNAL_AUTH_TOKEN = "test-internal-auth-token"; -const sharedRedisUrl = moduleDataUrl(` +const sharedRedisUrl = sharedRedisStubUrl(` export class RedisClient {} - export class WatchError extends Error {} export function redisDbFromEnv() { return 0; } `); const controlS3Url = moduleDataUrl(`export function makeS3Client() { return null; }`); @@ -31,10 +31,45 @@ const sharedAuthRolesUrl = moduleDataUrl(`export function validatePrincipalShape const sharedBoundedBodyUrl = repositoryFileUrl("shared/bounded-body.js"); const sharedErrorsUrl = repositoryFileUrl("shared/errors.js"); const sharedRandomIdUrl = repositoryFileUrl("shared/random-id.js"); +const sharedRedisLockUrl = moduleDataUrl(applyModuleReplacements( + readRepositoryFile("shared/redis-lock.js"), + [[/from "shared-random-id"/g, `from ${JSON.stringify(sharedRandomIdUrl)}`]] +)); const sharedQueueKeysUrl = moduleDataUrl(`export function queueStreamKey() { return ""; }`); const sharedRespondUrl = moduleDataUrl(readRepositoryFile("shared/respond.js")); const controlLibUrl = moduleDataUrl(`export function deleteLockKey(ns, worker) { return \`worker-delete-lock:\${ns}:\${worker}\`; }`); const sharedS3CleanupLifecycleUrl = repositoryFileUrl("shared/s3-cleanup-lifecycle.js"); +const sharedVersionUrl = repositoryFileUrl("shared/version.js"); +const sharedOptimisticRetryUrl = repositoryFileUrl("shared/optimistic-retry.js"); +const controlErrorsUrl = moduleDataUrl(applyModuleReplacements( + readRepositoryFile("control/errors.js"), + [ + [/from "shared-errors"/g, `from ${JSON.stringify(sharedErrorsUrl)}`], + [/from "shared-respond"/g, `from ${JSON.stringify(sharedRespondUrl)}`], + ] +)); +const controlWorkflowsClientUrl = moduleDataUrl(applyModuleReplacements( + readRepositoryFile("control/workflows-client.js"), + [ + [/from "shared-errors"/g, `from ${JSON.stringify(sharedErrorsUrl)}`], + [/from "control-errors"/g, `from ${JSON.stringify(controlErrorsUrl)}`], + ] +)); +const { postWorkflowsInternalRequest } = await import(controlWorkflowsClientUrl); +const controlOptimisticUrl = moduleDataUrl(applyModuleReplacements( + readRepositoryFile("control/optimistic.js"), + [ + [/from "shared-redis"/g, `from ${JSON.stringify(sharedRedisUrl)}`], + [/from "shared-optimistic-retry"/g, `from ${JSON.stringify(sharedOptimisticRetryUrl)}`], + ] +)); +const controlJsonBodyUrl = moduleDataUrl(applyModuleReplacements( + readRepositoryFile("control/json-body.js"), + [ + [/from "shared-bounded-body"/g, `from ${JSON.stringify(sharedBoundedBodyUrl)}`], + [/from "shared-respond"/g, `from ${JSON.stringify(sharedRespondUrl)}`], + ] +)); // Use the same stub constructor imported by control/shared.js so // runOptimistic's `instanceof WatchError` check observes the test error. const { WatchError: ControlSharedWatchError } = await import(sharedRedisUrl); @@ -48,19 +83,27 @@ const rewritten = applyModuleReplacements(readRepositoryFile("control/shared.js" [/from "shared-bounded-body"/g, `from ${JSON.stringify(sharedBoundedBodyUrl)}`], [/from "shared-errors"/g, `from ${JSON.stringify(sharedErrorsUrl)}`], [/from "shared-random-id"/g, `from ${JSON.stringify(sharedRandomIdUrl)}`], + [/from "shared-redis-lock"/g, `from ${JSON.stringify(sharedRedisLockUrl)}`], [/from "shared-queue-keys"/g, `from ${JSON.stringify(sharedQueueKeysUrl)}`], [/from "shared-respond"/g, `from ${JSON.stringify(sharedRespondUrl)}`], [/from "control-lib"/g, `from ${JSON.stringify(controlLibUrl)}`], [/from "shared-s3-cleanup-lifecycle"/g, `from ${JSON.stringify(sharedS3CleanupLifecycleUrl)}`], + [/from "shared-version"/g, `from ${JSON.stringify(sharedVersionUrl)}`], [/from "shared-internal-auth"/g, `from ${JSON.stringify(sharedInternalAuthUrl())}`], [/from "shared-observability"/g, `from ${JSON.stringify(OBSERVABILITY_NOOP_URL)}`], + [/from "control-workflows-client"/g, `from ${JSON.stringify(controlWorkflowsClientUrl)}`], + [/from "control-errors"/g, `from ${JSON.stringify(controlErrorsUrl)}`], + [/from "control-optimistic"/g, `from ${JSON.stringify(controlOptimisticUrl)}`], + [/from "control-json-body"/g, `from ${JSON.stringify(controlJsonBodyUrl)}`], ]); const { authErrorBody, authPolicyResponse, + acquireDeleteLock, assertWorkflowDeleteAllowed, cleanupDoAlarmsForWorker, + codedErrorLogFields, codedErrorResponse, ControlAbort, controlAbortResponse, @@ -69,6 +112,7 @@ const { rebuildDeclaredHostIndexes, releaseDeleteLock, runOptimistic, + secretEnvelopeErrorResponse, state, } = await import(moduleDataUrl(rewritten)); @@ -558,7 +602,12 @@ test("assertWorkflowDeleteAllowed hides transport diagnostics from response deta }; await assert.rejects( - () => assertWorkflowDeleteAllowed({ ns: "demo", worker: "api", version: "v2" }), + () => assertWorkflowDeleteAllowed({ + ns: "demo", + worker: "api", + version: "v2", + requestId: "rid-lifecycle-failure", + }), (err) => { assert.ok(err instanceof ControlAbort); const abort = /** @type {InstanceType} */ (err); @@ -576,6 +625,7 @@ test("assertWorkflowDeleteAllowed hides transport diagnostics from response deta namespace: "demo", worker: "api", version: "v2", + request_id: "rid-lifecycle-failure", error_message: "connect ECONNREFUSED workflows", }, }); @@ -612,21 +662,21 @@ test("assertWorkflowDeleteAllowed preserves active workflow blockers", async (t) ); }); -test("cleanupDoAlarmsForWorker uses a bounded workflows fetch signal", async (t) => { +test("shared workflows calls preserve endpoint-specific timeout behavior", async (t) => { restoreControlSharedStateAfter(t); state.env = { WDL_INTERNAL_AUTH_TOKEN: TEST_INTERNAL_AUTH_TOKEN }; - /** @type {number | null} */ - let timeoutMs = null; + /** @type {number[]} */ + const timeoutMs = []; /** @type {AbortSignal[]} */ const timeoutSignals = []; /** @type {ReturnType[]} */ const timeoutHandles = []; - /** @type {AbortSignal | undefined} */ - let fetchSignal; - /** @type {unknown} */ - let cleanupBody = null; + /** @type {AbortSignal[]} */ + const fetchSignals = []; + /** @type {Record} */ + const requestBodies = {}; const restoreTimeout = installMockProperty(AbortSignal, "timeout", (ms) => { - timeoutMs = ms; + timeoutMs.push(ms); const controller = new AbortController(); const handle = setTimeout(() => { controller.abort(new DOMException("The operation timed out.", "TimeoutError")); @@ -637,28 +687,67 @@ test("cleanupDoAlarmsForWorker uses a bounded workflows fetch signal", async (t) }); state.workflows = { /** - * @param {RequestInfo | URL} _url + * @param {RequestInfo | URL} url * @param {RequestInit | undefined} init */ - async fetch(_url, init) { - fetchSignal = init?.signal ?? undefined; + async fetch(url, init) { assert.equal(new Headers(init?.headers).get("x-wdl-internal-auth"), TEST_INTERNAL_AUTH_TOKEN); - cleanupBody = parseJsonObjectRequestBody(init, "DO alarm cleanup request body"); + const endpoint = String(url); + if (endpoint.endsWith("/lifecycle/check-delete")) { + assert.equal(new Headers(init?.headers).get("x-request-id"), "rid-lifecycle"); + assert.equal(init?.signal, undefined); + requestBodies.lifecycle = parseJsonObjectRequestBody(init, "workflow lifecycle request body"); + return Response.json({ allowed: true }); + } + assert.ok(init?.signal instanceof AbortSignal); + assert.equal(new Headers(init?.headers).get("x-request-id"), "rid-cleanup"); + fetchSignals.push(init.signal); + requestBodies.cleanup = parseJsonObjectRequestBody(init, "DO alarm cleanup request body"); return Response.json({ ok: true }); }, }; try { - await cleanupDoAlarmsForWorker({ ns: "demo", worker: "api", doStorageId: "do_old" }); + await assertWorkflowDeleteAllowed({ + ns: "demo", worker: "api", version: "v2", requestId: "rid-lifecycle", + }); + await cleanupDoAlarmsForWorker({ + ns: "demo", worker: "api", doStorageId: "do_old", requestId: "rid-cleanup", + }); } finally { for (const handle of timeoutHandles) clearTimeout(handle); restoreTimeout(); } - assert.equal(timeoutMs, 5_000); + assert.deepEqual(timeoutMs, [5_000]); assert.equal(timeoutSignals.length, 1); - assert.equal(timeoutSignals[0], fetchSignal); - assert.deepEqual(cleanupBody, { ns: "demo", worker: "api", doStorageId: "do_old" }); + assert.deepEqual(timeoutSignals, fetchSignals); + assert.deepEqual(requestBodies, { + lifecycle: { ns: "demo", worker: "api", version: "v2" }, + cleanup: { ns: "demo", worker: "api", doStorageId: "do_old" }, + }); +}); + +test("workflows transport requires an explicit timeout selection at runtime", async () => { + let fetchCalls = 0; + await assert.rejects( + postWorkflowsInternalRequest({ + workflows: { + async fetch() { + fetchCalls += 1; + return Response.json({ ok: true }); + }, + }, + headers: () => ({ "content-type": "application/json" }), + endpoint: "workflows/test", + body: {}, + logEvent: "workflow_test_failed", + timeoutMs: /** @type {any} */ (undefined), + makeError: (/** @type {"unavailable" | "request_failed"} */ failure) => new Error(failure), + }), + /request_failed/ + ); + assert.equal(fetchCalls, 0); }); test("codedErrorResponse preserves semantic status/code with a fallback code", async () => { @@ -674,7 +763,7 @@ test("codedErrorResponse preserves semantic status/code with a fallback code", a }); }); -test("codedErrorResponse falls back to detail message for non-ControlAbort coded errors", async () => { +test("codedErrorResponse hides diagnostic messages on coded server errors", async () => { const err = { status: 503, code: "workflow_backend_invalid_response", @@ -688,10 +777,95 @@ test("codedErrorResponse falls back to detail message for non-ControlAbort coded await assertJsonResponse(res, 503, { upstreamStatus: 200, error: "workflow_backend_invalid_response", - message: "Workflow backend returned a malformed response", + message: "Internal error", }); }); +test("codedErrorResponse keeps safe server context but strips diagnostic detail fields", async () => { + const err = new ControlAbort(500, "corrupt_meta", { + namespace: "demo", + worker: "api", + stage: "retained_meta_parse", + detail: "Unexpected token near secret bytes", + error_detail: "provider diagnostic", + }); + const res = controlAbortResponse(err); + + await assertJsonResponse(res, 500, { + namespace: "demo", + worker: "api", + error: "corrupt_meta", + message: "Internal error", + }); +}); + +test("codedErrorLogFields preserves bounded server diagnostics at error level callers", () => { + const err = new ControlAbort(500, "corrupt_meta", { + message: "Corrupt __meta__ for demo/api/v2", + version: "v2", + stage: "bundle_meta_parse", + detail: "__meta__ is not valid JSON", + }); + + assert.deepEqual(codedErrorLogFields(err), { + status: 500, + reason: "corrupt_meta", + error_message: "Corrupt __meta__ for demo/api/v2", + metadata_version: "v2", + stage: "bundle_meta_parse", + error_detail: "__meta__ is not valid JSON", + }); +}); + +test("codedErrorLogFields bounds structured diagnostic strings", () => { + const longValue = "x".repeat(4096); + const err = new ControlAbort(500, longValue, { + message: longValue, + version: longValue, + stage: longValue, + detail: longValue, + }); + + const fields = codedErrorLogFields(err, err.code, { + context: { ...err.details, safe_context: "kept" }, + }); + assert.equal(fields.safe_context, "kept"); + for (const alias of ["message", "detail", "version"]) { + assert.equal(Object.hasOwn(fields, alias), false, alias); + } + for (const key of ["reason", "error_message", "metadata_version", "stage", "error_detail"]) { + assert.equal(/** @type {string} */ (fields[key]).length, 2048, key); + assert.match(/** @type {string} */ (fields[key]), /\.\.\.$/, key); + } +}); + +test("secretEnvelopeErrorResponse bounds the final structured log diagnostics", async () => { + const longValue = "x".repeat(4096); + /** @type {Array<{ level: string, event: string, fields: Record }>} */ + const logs = []; + const err = Object.assign(new Error(longValue), { code: "secret_provider_error" }); + + const response = secretEnvelopeErrorResponse({ + err: /** @type {any} */ (err), + log(/** @type {string} */ level, /** @type {string} */ event, /** @type {Record} */ fields) { + logs.push({ level, event, fields }); + }, + event: "secret_mutation_rejected", + fields: { request_id: "rid-long-diagnostic" }, + }); + + await assertJsonResponse(response, 503, { + error: "secret_provider_error", + message: "Internal error", + }); + assert.equal(logs.length, 1); + const fields = logs[0].fields; + assert.equal(/** @type {string} */ (fields.error_message).length, 2048); + assert.equal(/** @type {string} */ (fields.error_detail).length, 2048); + assert.match(/** @type {string} */ (fields.error_message), /\.\.\.$/); + assert.match(/** @type {string} */ (fields.error_detail), /\.\.\.$/); +}); + test("codedErrorResponse strips only top-level wire-reserved detail fields", async () => { const err = Object.assign(new Error("secret nope"), { status: 403, @@ -802,6 +976,25 @@ test("readJsonBody: streamed body over maxBytes fails while reading", async () = }); }); +test("acquireDeleteLock stores a kind-prefixed random token", async () => { + /** @type {unknown[][]} */ + const calls = []; + const token = await acquireDeleteLock({ + /** @param {unknown[]} args */ + async set(...args) { + calls.push(args); + return "OK"; + }, + }, "demo", "api", "version"); + + assert.match(token || "", /^version:[0-9a-f]{32}$/); + assert.deepEqual(calls, [[ + "worker-delete-lock:demo:api", + token, + { nx: true, ttl: 30 }, + ]]); +}); + test("releaseDeleteLock uses token-scoped DELIFEQ", async (t) => { restoreControlSharedStateAfter(t); /** @type {unknown[][]} */ @@ -816,8 +1009,8 @@ test("releaseDeleteLock uses token-scoped DELIFEQ", async (t) => { calls.push(args); return 1; }, - }, "demo", "api", "token-a", "rid-delete"); + }, "demo", "api", "whole:token-a", "rid-delete"); - assert.deepEqual(calls, [["worker-delete-lock:demo:api", "token-a"]]); + assert.deepEqual(calls, [["worker-delete-lock:demo:api", "whole:token-a"]]); assert.deepEqual(logs, []); }); diff --git a/tests/unit/cross-language-identity.test.js b/tests/unit/cross-language-identity.test.js index 4e0a99a..ed5e402 100644 --- a/tests/unit/cross-language-identity.test.js +++ b/tests/unit/cross-language-identity.test.js @@ -1,6 +1,7 @@ import assert from "node:assert/strict"; import { test } from "node:test"; import { + KV_ID_RE, QUEUE_NAME_RE, WORKER_NAME_RE, WORKFLOW_INSTANCE_ID_RE, @@ -28,5 +29,6 @@ test("cross-language identity grammar fixture matches JS owners", () => { assertIdentityCases("runtimeLoadNs", isValidRuntimeLoadNs); assertIdentityCases("workerNames", (value) => WORKER_NAME_RE.test(value)); assertIdentityCases("queueNames", (value) => QUEUE_NAME_RE.test(value)); + assertIdentityCases("kvIds", (value) => KV_ID_RE.test(value)); assertIdentityCases("workflowInstanceIds", (value) => WORKFLOW_INSTANCE_ID_RE.test(value)); }); diff --git a/tests/unit/d1-owner-client.test.js b/tests/unit/d1-owner-client.test.js index d2c8f91..4c80f50 100644 --- a/tests/unit/d1-owner-client.test.js +++ b/tests/unit/d1-owner-client.test.js @@ -86,6 +86,18 @@ test("D1 owner client classifies non-error forwarded HTTP status as ok", async ( assert.equal(D1_OWNER_CLIENT_TEST_STATE.logs.at(-1).level, "info"); }); +test("D1 owner client rejects invalid endpoints before authenticated forwarding", async () => { + await assert.rejects( + () => mod.forwardToOwner(query(), env(), { ...owner(), endpoint: "8.8.8.8:8787" }, "req-invalid", 0), + (err) => { + assert.equal(/** @type {{ status?: unknown }} */ (err).status, 503); + assert.equal(/** @type {{ code?: unknown }} */ (err).code, "owner-endpoint-invalid"); + return true; + } + ); + assert.equal(D1_OWNER_CLIENT_TEST_STATE.fetches.length, 0); +}); + test("D1 owner client maps post-forward transport failures to result-unknown", async () => { restoreFetch(); restoreFetch = installMockFetch(makeRecordingFetch(D1_OWNER_CLIENT_TEST_STATE.fetches, { diff --git a/tests/unit/d1-owner-registry-regressions.test.js b/tests/unit/d1-owner-registry-regressions.test.js index c54d0f6..322c795 100644 --- a/tests/unit/d1-owner-registry-regressions.test.js +++ b/tests/unit/d1-owner-registry-regressions.test.js @@ -8,6 +8,7 @@ import { assertCurrentOwnerWithLeaseBudget, drainOwnedDbs, drainConcurrency, + normalizeDatabases, ownerGenerationKeyOf, ownerKeyOf, rebalanceDatabase, @@ -19,6 +20,26 @@ import { beforeEach(resetD1OwnerRegistryTestState); +test("D1 owner registry: records stored under another database scope fail closed", async () => { + const [requested, misplaced] = normalizeDatabases([ + { namespace: "tenant-a", databaseId: "db-a" }, + { namespace: "tenant-b", databaseId: "db-b" }, + ]); + D1_TEST_STATE.registryStore.set(ownerKeyOf(requested.dbKey), JSON.stringify({ + ...misplaced, + taskId: "task-b", + endpoint: "d1-runtime-b:8787", + generation: 3, + leaseExpiresAt: Date.now() + 60_000, + })); + + await assert.rejects( + () => resolveDbOwner({ REDIS_ADDR: "redis:6379" }, requested), + /D1 owner record is invalid/ + ); + assert.deepEqual(D1_TEST_STATE.setCommands, []); +}); + test("D1 owner registry: same-task reclaim repairs a stale generation counter without advancing owner generation", async () => { const identity = { namespace: "tenant-a", databaseId: "db1", dbKey: "tenant-a:db1", slot: 7 }; const ownerKey = ownerKeyOf(identity.dbKey); @@ -26,7 +47,7 @@ test("D1 owner registry: same-task reclaim repairs a stale generation counter wi D1_TEST_STATE.registryStore.set(ownerKey, JSON.stringify({ ...identity, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 11, leaseExpiresAt: Date.now() + 60_000, })); @@ -81,7 +102,7 @@ test("D1 owner registry: same-task hot path does not refresh the owner lease", a const owner = { ...identity, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 11, leaseExpiresAt: Date.now() + 60_000, }; @@ -107,7 +128,7 @@ test("D1 owner registry: expired same-task owner is reclaimed before local dispa const expiredOwner = { ...identity, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 11, leaseExpiresAt: redisNow - 1, }; @@ -137,7 +158,7 @@ test("D1 owner registry: expired same-task owner is reclaimed before local dispa test("D1 owner registry: lost owner state cannot validate an old owner from another task", async () => { const identity = { namespace: "tenant-a", databaseId: "db1", dbKey: "tenant-a:db1", slot: 7 }; - D1_TEST_STATE.taskIdentity = { taskId: "task-b", endpoint: "task-b:8787" }; + D1_TEST_STATE.taskIdentity = { taskId: "task-b", endpoint: "d1-runtime-b:8787" }; const owner = await resolveDbOwner({ REDIS_ADDR: "redis:6379" }, identity); @@ -160,13 +181,13 @@ test("D1 owner registry: lost owner state cannot validate an old owner from anot test("D1 owner registry: same task reclaims generation one after owner state loss", async () => { const identity = { namespace: "tenant-a", databaseId: "db1", dbKey: "tenant-a:db1", slot: 7 }; - D1_TEST_STATE.taskIdentity = { taskId: "task-b", endpoint: "task-b:8787" }; + D1_TEST_STATE.taskIdentity = { taskId: "task-b", endpoint: "d1-runtime-b:8787" }; const owner = await resolveDbOwner({ REDIS_ADDR: "redis:6379" }, identity); assert.equal(owner.taskId, "task-b"); assert.equal(owner.generation, 1); - assert.equal(owner.endpoint, "task-b:8787"); + assert.equal(owner.endpoint, "d1-runtime-b:8787"); assert.deepEqual(await assertCurrentOwner({ REDIS_ADDR: "redis:6379" }, owner), owner); }); @@ -176,7 +197,7 @@ test("D1 owner registry: current-owner check rejects expired matching leases", a const owner = { ...identity, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 3, leaseExpiresAt: redisNow - 1, }; @@ -199,7 +220,7 @@ test("D1 owner registry: current-owner check renews leases inside the guard wind const owner = { ...identity, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 3, leaseExpiresAt: redisNow + 250, }; @@ -224,7 +245,7 @@ test("D1 owner registry: current-owner budget result carries Redis-time remainin const owner = { ...identity, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 3, leaseExpiresAt: redisNow + 5000, }; @@ -246,7 +267,7 @@ test("D1 owner registry: current-owner check rejects renewals that still miss th const owner = { ...identity, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 3, leaseExpiresAt: redisNow + 250, }; @@ -274,7 +295,7 @@ test("D1 owner registry: post-renew guard budget uses the renew Redis time", asy const owner = { ...identity, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 3, leaseExpiresAt: redisNow + 250, }; @@ -300,7 +321,7 @@ test("D1 owner registry: current-owner check uses Redis server time as lease aut const owner = { ...identity, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 3, leaseExpiresAt, }; @@ -324,7 +345,7 @@ test("D1 owner registry: stale generation repair failure is best-effort", async const owner = { ...identity, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 11, leaseExpiresAt: Date.now() + 60_000, }; @@ -352,7 +373,7 @@ test("D1 owner registry: claim race falls back to the winner owner", async () => const winner = { ...identity, taskId: "task-b", - endpoint: "task-b:8787", + endpoint: "d1-runtime-b:8787", generation: 4, leaseExpiresAt: Date.now() + 60_000, }; @@ -381,7 +402,7 @@ test("D1 owner registry: claim race waits briefly for a winner owner", async () const winner = { ...identity, taskId: "task-b", - endpoint: "task-b:8787", + endpoint: "d1-runtime-b:8787", generation: 4, leaseExpiresAt: Date.now() + 60_000, }; @@ -411,7 +432,7 @@ test("D1 owner registry: observed remote owner avoids repeated registry reads", const owner = { ...identity, taskId: "task-b", - endpoint: "task-b:8787", + endpoint: "d1-runtime-b:8787", generation: 4, leaseExpiresAt: Date.now() + 60_000, }; @@ -438,7 +459,7 @@ test("D1 owner registry: observed local owner avoids repeated registry reads", a const owner = { ...identity, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 4, leaseExpiresAt: Date.now() + 60_000, }; @@ -462,14 +483,14 @@ test("D1 owner registry: forced refresh bypasses observed owner cache", async () const firstOwner = { ...identity, taskId: "task-b", - endpoint: "task-b:8787", + endpoint: "d1-runtime-b:8787", generation: 4, leaseExpiresAt: Date.now() + 60_000, }; const nextOwner = { ...identity, taskId: "task-c", - endpoint: "task-c:8787", + endpoint: "d1-runtime-c:8787", generation: 5, leaseExpiresAt: Date.now() + 60_000, }; @@ -494,7 +515,7 @@ test("D1 owner registry: takeover bumps generation monotonically even when the s dbKey: "tenant-a:db1", slot: 7, taskId: "task-b", - endpoint: "task-b:8787", + endpoint: "d1-runtime-b:8787", generation: 11, leaseExpiresAt: redisNow - 1, }; @@ -520,7 +541,7 @@ test("D1 owner registry: drain waits for in-flight queries before releasing owne dbKey: "tenant-a:db1", slot: 7, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 3, leaseExpiresAt: Date.now() + 60_000, }; @@ -560,7 +581,7 @@ test("D1 owner registry: drain releases idle databases even when one actor times dbKey: `tenant-a:${databaseId}`, slot: idx, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 3, leaseExpiresAt: Date.now() + 60_000, })); @@ -614,7 +635,7 @@ test("D1 owner registry: takeover retries WatchError before surfacing an ownersh dbKey: "tenant-a:db1", slot: 7, taskId: "task-b", - endpoint: "task-b:8787", + endpoint: "d1-runtime-b:8787", generation: 4, leaseExpiresAt: redisNow - 1, }; @@ -641,7 +662,7 @@ test("D1 owner registry: rebalance retries WatchError before moving ownership", const currentOwner = { ...database, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 6, leaseExpiresAt: Date.now() + 60_000, }; @@ -655,7 +676,7 @@ test("D1 owner registry: rebalance retries WatchError before moving ownership", const result = await rebalanceDatabase( { REDIS_ADDR: "redis:6379" }, database, - { taskId: "task-b", endpoint: "task-b:8787" } + { taskId: "task-b", endpoint: "d1-runtime-b:8787" } ); assert.equal(result.outcome, "moved"); @@ -674,7 +695,7 @@ test("D1 owner registry: rebalance to the current owner is a no-op", async () => const currentOwner = { ...database, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 6, leaseExpiresAt: Date.now() + 60_000, }; @@ -687,7 +708,7 @@ test("D1 owner registry: rebalance to the current owner is a no-op", async () => const result = await rebalanceDatabase( { REDIS_ADDR: "redis:6379" }, database, - { taskId: "task-a", endpoint: "task-a:8787" } + { taskId: "task-a", endpoint: "d1-runtime-a:8787" } ); assert.equal(result.outcome, "unchanged"); @@ -709,7 +730,7 @@ test("D1 owner registry: rebalance to same task with a different endpoint refres const currentOwner = { ...database, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 6, leaseExpiresAt: Date.now() + 60_000, }; @@ -720,12 +741,12 @@ test("D1 owner registry: rebalance to same task with a different endpoint refres const result = await rebalanceDatabase( { REDIS_ADDR: "redis:6379" }, database, - { taskId: "task-a", endpoint: "task-a-new:8787" } + { taskId: "task-a", endpoint: "d1-runtime-a-new:8787" } ); assert.equal(result.outcome, "moved"); assert.equal(result.owner.taskId, "task-a"); - assert.equal(result.owner.endpoint, "task-a-new:8787"); + assert.equal(result.owner.endpoint, "d1-runtime-a-new:8787"); assert.equal(result.owner.generation, 7); assert.equal(D1_TEST_STATE.ownedDbs.has(database.dbKey), false); assert.deepEqual(D1_TEST_STATE.forgottenStorageSizes, [database.dbKey]); @@ -740,7 +761,7 @@ test("D1 owner registry: renew uses bounded concurrency instead of strict serial dbKey: `tenant-a:db${idx}`, slot: idx, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 1, leaseExpiresAt: Date.now() + 60_000, }; diff --git a/tests/unit/d1-owner-registry.test.js b/tests/unit/d1-owner-registry.test.js index 600a39e..aa38848 100644 --- a/tests/unit/d1-owner-registry.test.js +++ b/tests/unit/d1-owner-registry.test.js @@ -49,18 +49,26 @@ test("D1 owner registry: owner keys encode database keys safely", () => { }); test("D1 owner registry: parseOwner accepts Redis strings and bulk bytes", () => { + const [identity] = normalizeDatabases([{ namespace: "tenant-a", databaseId: "d1_main" }]); const owner = { - namespace: "tenant-a", - databaseId: "d1_main", - dbKey: "tenant-a:d1_main", + ...identity, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 3, leaseExpiresAt: Date.now() + 1000, }; - assert.deepEqual(parseOwner(JSON.stringify(owner)), owner); - assert.deepEqual(parseOwner(new TextEncoder().encode(JSON.stringify(owner))), owner); - assert.equal(parseOwner(null), null); + assert.deepEqual(parseOwner(JSON.stringify(owner), identity.dbKey), owner); + assert.deepEqual(parseOwner(new TextEncoder().encode(JSON.stringify(owner)), identity.dbKey), owner); + assert.equal(parseOwner(null, identity.dbKey), null); + assert.throws( + () => parseOwner(JSON.stringify({ ...owner, endpoint: "8.8.8.8:8787" }), identity.dbKey), + /D1 owner record is invalid/ + ); + assert.throws( + () => parseOwner(JSON.stringify({ ...owner, dbKey: "tenant-b:db-b" }), identity.dbKey), + /D1 owner record is invalid/ + ); + assert.throws(() => parseOwner("{not-json", identity.dbKey), /D1 owner record is invalid/); }); test("D1 owner registry: rebalance request normalization validates shape", () => { @@ -76,7 +84,15 @@ test("D1 owner registry: rebalance request normalization validates shape", () => assert.throws(() => normalizeDatabases([]), /databases must be a non-empty array/); assert.throws( () => normalizeDatabases([{ namespace: "tenant-a", databaseId: "db:bad" }]), - /databaseId must not contain ':'/ + /databaseId is invalid/ + ); + assert.throws( + () => normalizeDatabases([{ namespace: "admin", databaseId: "main" }]), + /namespace is invalid/ ); assert.throws(() => normalizeTarget({ taskId: "task-b" }), /target.endpoint is required/); + assert.throws( + () => normalizeTarget({ taskId: "task-b", endpoint: "8.8.8.8:8787" }), + /target.endpoint is invalid/ + ); }); diff --git a/tests/unit/d1-protocol.test.js b/tests/unit/d1-protocol.test.js index f60e2b7..a3c4bea 100644 --- a/tests/unit/d1-protocol.test.js +++ b/tests/unit/d1-protocol.test.js @@ -56,6 +56,12 @@ test("D1 protocol: db key includes namespace and database id", () => { assert.notEqual(dbKeyOf("tenant-a", "main"), dbKeyOf("tenant-b", "main")); }); +test("D1 protocol: db keys reject non-canonical runtime namespaces and database ids", () => { + assert.throws(() => dbKeyOf("admin", "main"), /namespace is invalid/); + assert.throws(() => dbKeyOf("tenant-a", "db/child"), /databaseId is invalid/); + assert.throws(() => dbKeyOf("tenant-a", "a".repeat(129)), /databaseId is invalid/); +}); + test("D1 protocol: slot hash is stable and bounded", () => { const slot = slotOf("tenant-a", "main", 128); assert.equal(slot, slotOf("tenant-a", "main", 128)); @@ -386,7 +392,12 @@ test("D1 protocol: invalid request shape throws protocol error", () => { }); test("D1 protocol: classifies user-facing errors by category", () => { - for (const code of ["owner-lease-too-short", "lease-budget-exhausted"]) { + for (const code of [ + "owner-record-invalid", + "owner-endpoint-invalid", + "owner-lease-too-short", + "lease-budget-exhausted", + ]) { assert.deepEqual( classifyD1Error(new D1ProtocolError(503, code, "lease budget low")), { diff --git a/tests/unit/d1-router.test.js b/tests/unit/d1-router.test.js index e9e8a59..b076945 100644 --- a/tests/unit/d1-router.test.js +++ b/tests/unit/d1-router.test.js @@ -7,7 +7,7 @@ import { assertJsonResponse, readJsonResponse } from "../helpers/response-json.j const taskIdentityUrl = moduleDataUrl(` -export async function resolveTaskIdentity() { return { taskId: "task-a", endpoint: "task-a:8787" }; } +export async function resolveTaskIdentity() { return { taskId: "task-a", endpoint: "d1-runtime-a:8787" }; } `); const timeoutUrl = moduleDataUrl(` export function createD1QueryDeadline() { @@ -20,7 +20,7 @@ export async function ownerLeaseExpiredByRedisTime(_env, owner) { } export async function resolveDbOwner(env, query, options) { return /** @type {any} */ (globalThis).__d1RouterResolveDbOwner?.(env, query, options) || - { taskId: "task-a", endpoint: "task-a:8787", generation: 1 }; + { taskId: "task-a", endpoint: "d1-runtime-a:8787", generation: 1 }; } export async function takeoverExpiredOwner(env, owner) { return /** @type {any} */ (globalThis).__d1RouterTakeoverExpiredOwner?.(env, owner) || owner; @@ -144,14 +144,14 @@ test("D1 router uses takeover owner even after refresh is disabled", async () => const oldOwner = { dbKey: query.dbKey, taskId: "task-b", - endpoint: "task-b:8787", + endpoint: "d1-runtime-b:8787", generation: 7, leaseExpiresAt: Date.now() - 1_000, }; const takeoverOwner = { dbKey: query.dbKey, taskId: "task-a", - endpoint: "task-a:8787", + endpoint: "d1-runtime-a:8787", generation: 8, leaseExpiresAt: Date.now() + 60_000, }; @@ -206,7 +206,7 @@ test("D1 router owner-not-ready errors do not expose owner task identity", async const owner = { dbKey: query.dbKey, taskId: "task-b", - endpoint: "task-b:8787", + endpoint: "d1-runtime-b:8787", generation: 7, leaseExpiresAt: Date.now() + 60_000, }; diff --git a/tests/unit/d1-task-identity.test.js b/tests/unit/d1-task-identity.test.js index f254edb..eef4487 100644 --- a/tests/unit/d1-task-identity.test.js +++ b/tests/unit/d1-task-identity.test.js @@ -6,6 +6,7 @@ import { applyModuleReplacements, moduleDataUrl, readRepositoryFile, + repositoryFileUrl, sharedModuleDataUrl, } from "../helpers/load-shared-module.js"; @@ -13,11 +14,16 @@ const { D1ProtocolError } = await loadD1Protocol(); const PROTOCOL_URL = d1ProtocolDataUrl(); const SHARED_TASK_IDENTITY_URL = sharedModuleDataUrl("shared/task-identity.js"); +const SHARED_OWNER_ENDPOINT_URL = repositoryFileUrl("shared/owner-endpoint.js"); const src = applyModuleReplacements(readRepositoryFile("d1-runtime/task-identity.js"), [ [ /import \{ createTaskIdentityResolver \} from "shared-task-identity";/, `import { createTaskIdentityResolver } from ${JSON.stringify(SHARED_TASK_IDENTITY_URL)};`, ], + [ + /import \{ validOwnerEndpointForService \} from "shared-owner-endpoint";/, + `import { validOwnerEndpointForService } from ${JSON.stringify(SHARED_OWNER_ENDPOINT_URL)};`, + ], [ /import \{ D1ProtocolError \} from "d1-runtime-protocol";/, `import { D1ProtocolError } from ${JSON.stringify(PROTOCOL_URL)};`, @@ -68,6 +74,13 @@ test("D1 task identity: env task id and endpoint win", async () => { }); }); +test("D1 task identity: public owner endpoints are rejected", () => { + assert.throws( + () => taskIdentityFromEnv({ D1_TASK_ID: "task-a", D1_TASK_ENDPOINT: "8.8.8.8:8787" }), + (err) => isProtocolError(err, 503, "task-identity-unavailable") + ); +}); + test("D1 task identity: partial env identity is rejected", () => { assert.throws( () => taskIdentityFromEnv({ D1_TASK_ID: "task-only" }), @@ -85,11 +98,11 @@ test("D1 task identity: ECS metadata yields task arn and private IPv4 endpoint", Networks: [{ IPv4Addresses: ["10.0.42.17"] }], }, ], - }, { D1_TASK_PORT: "9797" }); + }); assert.deepEqual(identity, { taskId: "arn:aws:ecs:us-east-1:123456789012:task/cluster/abc", - endpoint: "10.0.42.17:9797", + endpoint: "10.0.42.17:8787", source: "ecs-metadata", }); }); diff --git a/tests/unit/do-alarm-client.test.js b/tests/unit/do-alarm-client.test.js index 34eebf3..8a7cd81 100644 --- a/tests/unit/do-alarm-client.test.js +++ b/tests/unit/do-alarm-client.test.js @@ -22,6 +22,9 @@ export class DoRuntimeError extends Error { export function nonEmptyAlarmString(value) { return typeof value === "string" && value.length > 0 ? value : null; } +export function isWellFormedUnicodeString(value) { + return typeof value === "string" && value.isWellFormed(); +} `); const SHARED_INTERNAL_AUTH_URL = sharedInternalAuthUrl(); const TEST_INTERNAL_AUTH_TOKEN = "test-internal-auth-token"; @@ -194,5 +197,18 @@ test("alarm input validation remains local before backend calls", async () => { ), /retryCount must be a non-negative integer/, ); + await assert.rejects( + () => mod.setAlarmIndex( + alarmEnv({ WORKFLOWS_BACKEND: workflowsBackend() }), + props, + { + className: "Room", + objectName: "\ud800", + scheduledTime: 123456, + token: "row-token", + }, + ), + /objectName must contain well-formed Unicode/, + ); assert.equal(calls.length, 0); }); diff --git a/tests/unit/do-alarm-shim.test.js b/tests/unit/do-alarm-shim.test.js index 02f9f5e..7140409 100644 --- a/tests/unit/do-alarm-shim.test.js +++ b/tests/unit/do-alarm-shim.test.js @@ -1,15 +1,342 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { DO_ALARM_SHIM_SOURCE } from "../../do-runtime/alarm-shim-source.js"; +import { + DO_ALARM_SHIM_SOURCE, +} from "../../do-runtime/alarm-shim-source.js"; import { makeDoAlarmBinding, makeDoAlarmStorage } from "../helpers/do-alarm-shim-fixture.js"; -import { moduleDataUrl } from "../helpers/load-shared-module.js"; +import { applyModuleReplacements, moduleDataUrl } from "../helpers/load-shared-module.js"; +import { withMockedGlobal, withMockedProperty } from "../helpers/mock-global.js"; import { assertJsonResponse } from "../helpers/response-json.js"; -const shimSource = DO_ALARM_SHIM_SOURCE.replace( - "function wrapStorage", - "export function wrapStorage" -); -const { wrapDurableObjectClass, wrapStorage } = await import(moduleDataUrl(shimSource)); +const shimSource = applyModuleReplacements(DO_ALARM_SHIM_SOURCE, [ + ["function wrapStorage", "export function wrapStorage"], + ["function formatWrappedError", "export function formatWrappedError"], +]); +const { formatWrappedError, wrapDurableObjectClass, wrapStorage } = await import(moduleDataUrl(shimSource)); + +test("DO alarm shim: error formatting cannot replace the original failure", () => { + const hostile = new Error("original"); + Object.defineProperties(hostile, { + name: { get() { throw new Error("name getter failed"); } }, + message: { get() { throw new Error("message getter failed"); } }, + code: { get() { throw new Error("code getter failed"); } }, + }); + assert.deepEqual(formatWrappedError(hostile), { + error_name: "Error", + error_message: "Unknown error", + }); + + const proxy = new Proxy({}, { + get() { throw new Error("proxy getter failed"); }, + getPrototypeOf() { throw new Error("prototype trap failed"); }, + }); + assert.deepEqual(formatWrappedError(proxy), { + error_name: "Error", + error_message: "Unknown error", + }); +}); + +test("DO alarm shim: internal RPC dispatch invokes tenant methods without adding arguments", async () => { + const { storage } = makeDoAlarmStorage(); + class RpcRoom { + /** @param {{ storage: unknown }} ctx */ + constructor(ctx) { + this.ctx = ctx; + } + async inspect(/** @type {unknown} */ value) { + await Promise.resolve(); + return { value }; + } + } + const Wrapped = wrapDurableObjectClass(RpcRoom, "Room"); + const instance = new Wrapped( + { storage, id: "alice" }, + { __WDL_DO_ALARMS__: makeDoAlarmBinding([]) } + ); + /** @param {string} method */ + const request = (method) => new Request("https://do.internal/__wdl_rpc", { + method: "POST", + headers: { "x-wdl-do-internal-rpc": "1", "x-request-id": "rid-rpc" }, + body: JSON.stringify({ method, args: ["value"] }), + }); + await assertJsonResponse(await instance.fetch(request("inspect")), 200, { + ok: true, + result: { + value: "value", + }, + }); + await assertJsonResponse(await instance.fetch(request("missing")), 404, { + error: "do_rpc_method_not_found", + message: "Durable Object RPC method missing was not found", + }); +}); + +test("DO alarm shim: repair logging cannot replace the stored alarm result", async () => { + /** @param {(read: () => Promise) => Promise} callback */ + async function readWithBrokenLogDependency(callback) { + const { storage } = makeDoAlarmStorage({ + scheduled_time: 1234, + retry_count: 0, + in_flight: 0, + token: "sqlite-token", + }); + const alarmBinding = makeDoAlarmBinding([]); + alarmBinding.setAlarmIndex = async () => { + throw new Error("backend unavailable"); + }; + const wrapped = wrapStorage(storage, alarmBinding, "Room", "alice"); + await callback(async () => assert.equal(await wrapped.getAlarm(), 1234)); + } + + await readWithBrokenLogDependency((read) => + withMockedGlobal("Date", new Proxy(Date, { + construct() { + throw new Error("tenant Date"); + }, + }), read)); + await readWithBrokenLogDependency((read) => + withMockedProperty(JSON, "stringify", () => { + throw new Error("tenant stringify"); + }, read)); + await readWithBrokenLogDependency((read) => + withMockedProperty(console, "log", () => { + throw new Error("tenant console"); + }, read)); +}); + +test("DO alarm shim: internal alarm dispatch ignores tenant-patched request intrinsics", async () => { + /** @type {unknown[][]} */ + const calls = []; + const { storage, state } = makeDoAlarmStorage({ + scheduled_time: Date.now() - 1000, + retry_count: 0, + in_flight: 0, + token: "captured-intrinsics-token", + }); + class AlarmCounter { + /** @param {{ storage: unknown }} ctx */ + constructor(ctx) { + this.ctx = ctx; + this.alarms = 0; + } + async alarm() { + this.alarms += 1; + } + async fetch() { + return new Response("tenant fetch"); + } + } + const Wrapped = wrapDurableObjectClass(AlarmCounter, "Room"); + const instance = new Wrapped( + { storage, id: "alice" }, + { __WDL_DO_ALARMS__: makeDoAlarmBinding(calls) } + ); + const request = new Request("https://do.internal/__wdl_alarm", { + method: "POST", + headers: { "x-wdl-do-internal-alarm": "1" }, + body: JSON.stringify({ token: "captured-intrinsics-token", retryCount: 0 }), + }); + + await withMockedProperty(Headers.prototype, "get", () => null, async () => { + await withMockedProperty(Request.prototype, "json", async () => { + throw new Error("tenant Request.json"); + }, async () => { + await withMockedProperty(Response, "json", () => { + throw new Error("tenant Response.json"); + }, async () => { + const response = await instance.fetch(request); + await assertJsonResponse(response, 200, { ok: true }); + }); + }); + }); + + assert.equal(instance.alarms, 1); + assert.equal(state.row, null); +}); + +test("DO alarm shim: class-field fetch cannot intercept internal alarm dispatch", async () => { + const { storage, state } = makeDoAlarmStorage({ + scheduled_time: Date.now() - 1000, + retry_count: 0, + in_flight: 0, + token: "class-field-fetch-token", + }); + class ClassFieldFetch { + alarms = 0; + fetchCalls = 0; + ["fetch"] = async () => { + this.fetchCalls += 1; + return new Response("tenant fetch"); + }; + /** @param {{ storage: unknown }} ctx */ + constructor(ctx) { + this.ctx = ctx; + } + async alarm() { + this.alarms += 1; + } + } + const Wrapped = wrapDurableObjectClass(ClassFieldFetch, "Room"); + const instance = new Wrapped( + { storage, id: "alice" }, + { __WDL_DO_ALARMS__: makeDoAlarmBinding([]) } + ); + const alarmResponse = await instance.fetch(new Request("https://do.internal/__wdl_alarm", { + method: "POST", + headers: { "x-wdl-do-internal-alarm": "1" }, + body: JSON.stringify({ token: "class-field-fetch-token", retryCount: 0 }), + })); + await assertJsonResponse(alarmResponse, 200, { ok: true }); + assert.equal(instance.alarms, 1); + assert.equal(instance.fetchCalls, 0); + assert.equal(state.row, null); + + assert.equal(await (await instance.fetch(new Request("https://do.internal/tenant"))).text(), "tenant fetch"); + assert.equal(instance.fetchCalls, 1); +}); + +test("DO alarm shim: class-field alarm executes before its row is cleared", async () => { + const { storage, state } = makeDoAlarmStorage({ + scheduled_time: Date.now() - 1000, + retry_count: 2, + in_flight: 0, + token: "class-field-alarm-token", + }); + class ClassFieldAlarm { + alarms = 0; + /** @type {number | null} */ + retryCount = null; + /** @param {{ retryCount: number }} info */ + alarm = async (info) => { + this.alarms += 1; + this.retryCount = info.retryCount; + }; + /** @param {{ storage: unknown }} ctx */ + constructor(ctx) { + this.ctx = ctx; + } + } + const Wrapped = wrapDurableObjectClass(ClassFieldAlarm, "Room"); + const instance = new Wrapped( + { storage, id: "alice" }, + { __WDL_DO_ALARMS__: makeDoAlarmBinding([]) } + ); + + const response = await instance.fetch(new Request("https://do.internal/__wdl_alarm", { + method: "POST", + headers: { "x-wdl-do-internal-alarm": "1" }, + body: JSON.stringify({ token: "class-field-alarm-token", retryCount: 2 }), + })); + + await assertJsonResponse(response, 200, { ok: true }); + assert.equal(instance.alarms, 1); + assert.equal(instance.retryCount, 2); + assert.equal(state.row, null); +}); + +test("DO alarm shim: own accessors retain their instance receiver", async () => { + const { storage, state } = makeDoAlarmStorage({ + scheduled_time: Date.now() - 1000, + retry_count: 0, + in_flight: 0, + token: "accessor-alarm-token", + }); + class AccessorHandlers { + #alarms = 0; + /** @param {{ storage: unknown }} ctx */ + constructor(ctx) { + this.ctx = ctx; + Object.defineProperties(this, { + fetch: { + configurable: true, + get: () => async () => new Response(String(this.#alarms)), + }, + alarm: { + configurable: true, + get: () => async () => { + this.#alarms += 1; + }, + }, + }); + } + } + const Wrapped = wrapDurableObjectClass(AccessorHandlers, "Room"); + const instance = new Wrapped( + { storage, id: "alice" }, + { __WDL_DO_ALARMS__: makeDoAlarmBinding([]) } + ); + + const alarmResponse = await instance.fetch(new Request("https://do.internal/__wdl_alarm", { + method: "POST", + headers: { "x-wdl-do-internal-alarm": "1" }, + body: JSON.stringify({ token: "accessor-alarm-token", retryCount: 0 }), + })); + await assertJsonResponse(alarmResponse, 200, { ok: true }); + assert.equal(state.row, null); + assert.equal(await (await instance.fetch(new Request("https://do.internal/status"))).text(), "1"); +}); + +test("DO alarm shim: alarm getter remains lazy until internal alarm dispatch", async () => { + class LazyAlarmGetter { + /** @param {{ storage: unknown }} ctx */ + constructor(ctx) { + this.ctx = ctx; + } + async fetch() { + return new Response("tenant fetch"); + } + get alarm() { + throw new Error("alarm getter must remain lazy"); + } + } + const { storage } = makeDoAlarmStorage(); + const Wrapped = wrapDurableObjectClass(LazyAlarmGetter, "Room"); + const instance = new Wrapped( + { storage, id: "alice" }, + { __WDL_DO_ALARMS__: makeDoAlarmBinding([]) } + ); + + const response = await instance.fetch(new Request("https://do.internal/tenant")); + assert.equal(await response.text(), "tenant fetch"); +}); + +test("DO alarm shim: storage facade ignores tenant-patched proxy intrinsics", async () => { + const { storage } = makeDoAlarmStorage({ + scheduled_time: 1234, + retry_count: 0, + in_flight: 0, + token: "captured-proxy-token", + }); + const alarmBinding = makeDoAlarmBinding([]); + const hostileProxy = new Proxy(Proxy, { + construct() { + throw new Error("tenant Proxy"); + }, + }); + + await withMockedGlobal("Proxy", hostileProxy, async () => { + await withMockedProperty(Reflect, "get", () => { + throw new Error("tenant Reflect.get"); + }, async () => { + await withMockedProperty(Reflect, "apply", () => { + throw new Error("tenant Reflect.apply"); + }, async () => { + class AlarmReader { + /** @param {{ storage: unknown }} ctx */ + constructor(ctx) { + this.ctx = ctx; + } + } + const Wrapped = wrapDurableObjectClass(AlarmReader, "Room"); + const instance = new Wrapped( + { storage, id: "alice" }, + { __WDL_DO_ALARMS__: alarmBinding } + ); + assert.equal(await instance.ctx.storage.getAlarm(), 1234); + }); + }); + }); +}); test("DO alarm shim: transaction setAlarm then deleteAlarm flushes only the final delete", async () => { /** @type {unknown[][]} */ @@ -501,3 +828,58 @@ test("DO alarm shim: deleteAll deleteAlarm false preserves alarm row without bac assert.deepEqual(state.row, { ...initial, last_error: null }); assert.equal(kv.size, 0); }); + +test("DO alarm shim: deleteAll ignores tenant-patched array iteration", async () => { + /** @type {unknown[][]} */ + const calls = []; + /** @type {string[]} */ + const dropped = []; + const initial = { + scheduled_time: 1234, + retry_count: 0, + in_flight: 0, + token: "preserve-token", + }; + const { storage, state } = makeDoAlarmStorage(initial); + const originalExec = storage.sql.exec; + /** @param {string} statement @param {...unknown} params */ + storage.sql.exec = function exec(statement, ...params) { + if (statement.startsWith("SELECT type, name FROM sqlite_master")) { + return [ + { type: "table", name: "tenant_table" }, + { type: "table", name: "_wdl_do_alarms" }, + ]; + } + if (statement === 'DROP TABLE IF EXISTS "tenant_table"') { + dropped.push(statement); + return []; + } + return Reflect.apply(originalExec, this, [statement, ...params]); + }; + const wrapped = wrapStorage(storage, makeDoAlarmBinding(calls), "Room", "alice"); + const originalIterator = Array.prototype[Symbol.iterator]; + + await withMockedProperty(Map.prototype, "keys", () => { + throw new Error("tenant Map.keys"); + }, () => withMockedProperty( + Array.prototype, + Symbol.iterator, + /** @this {any[]} */ function hostileIterator() { + const first = this[0]; + if ( + (this.length === 1 && first?.deleteAlarm === false) || + (first && typeof first === "object" && "type" in first) + ) { + return Reflect.apply(originalIterator, [], []); + } + return Reflect.apply(originalIterator, this, []); + }, + async () => { + await wrapped.deleteAll({ deleteAlarm: false }); + } + )); + + assert.deepEqual(dropped, ['DROP TABLE IF EXISTS "tenant_table"']); + assert.deepEqual(calls, []); + assert.deepEqual(state.row, { ...initial, last_error: null }); +}); diff --git a/tests/unit/do-owner-client.test.js b/tests/unit/do-owner-client.test.js index 951c04c..09f8e3f 100644 --- a/tests/unit/do-owner-client.test.js +++ b/tests/unit/do-owner-client.test.js @@ -103,6 +103,14 @@ test("DO owner client forwards invoke requests with owner fence and hop headers" assert.equal(DO_OWNER_CLIENT_TEST_STATE.logs.at(-1).fields.path, "/internal/do/storage/delete"); }); +test("DO owner client rejects invalid endpoints before authenticated forwarding", async () => { + await assert.rejects( + mod.forwardToOwner(invoke(), env(), { ...owner(), endpoint: "8.8.8.8:8788" }, "req-invalid", 0), + (err) => doErrorMatches(err, { status: 503, code: "owner_unavailable" }) + ); + assert.equal(DO_OWNER_CLIENT_TEST_STATE.fetches.length, 0); +}); + test("DO owner client maps exhausted forward hops to a stable 503 code", async () => { await assert.rejects( mod.forwardToOwner(invoke(), env(), owner(), "req-2", 2), diff --git a/tests/unit/do-owner-registry.test.js b/tests/unit/do-owner-registry.test.js index dd21a5d..5ed2cec 100644 --- a/tests/unit/do-owner-registry.test.js +++ b/tests/unit/do-owner-registry.test.js @@ -10,6 +10,7 @@ import { ownerGenerationKeyOf, ownerLeaseGuardMs, ownerKeyOf, + parseOwner, renewOwnedScopes, releaseOwner, resetDoOwnerRegistryTestState, @@ -33,6 +34,7 @@ const INVALID_RENEW_TIME_BASES = [1234.5, NaN, Infinity, -Infinity, -1]; const DO_STORAGE_ID = "do_0123456789abcdef0123456789abcdef"; const OWNER_KEY = `${DO_STORAGE_ID}:Room:shard0`; const STORAGE_POINTER_KEY = "worker:do-storage:tenant:chat"; +const DELETE_LOCK_KEY = "worker-delete-lock:tenant:chat"; beforeEach(resetDoOwnerRegistryTestState); @@ -60,13 +62,37 @@ function ownerRecord(overrides = {}) { worker: "chat", doStorageId: DO_STORAGE_ID, taskId: "task-a", - endpoint: "task-a:8788", + endpoint: "do-runtime-a:8788", generation: 7, leaseExpiresAt: Date.now() + LEASE_DURATION_MS, ...overrides, }; } +test("DO owner registry rejects malformed persisted owner endpoints", () => { + const owner = ownerRecord(); + assert.deepEqual(parseOwner(JSON.stringify(owner), OWNER_KEY), owner); + assert.equal(parseOwner(null, OWNER_KEY), null); + assert.throws( + () => parseOwner(JSON.stringify(ownerRecord({ endpoint: "8.8.8.8:8788" })), OWNER_KEY), + /DO owner record is invalid/ + ); + assert.throws( + () => parseOwner(JSON.stringify(ownerRecord({ ownerKey: `${DO_STORAGE_ID}:Room:shard1` })), OWNER_KEY), + /DO owner record is invalid/ + ); + for (const invalidScope of [ + { ns: "tenant:other" }, + { worker: "chat:other" }, + ]) { + assert.throws( + () => parseOwner(JSON.stringify(ownerRecord(invalidScope)), OWNER_KEY), + /DO owner record is invalid/ + ); + } + assert.throws(() => parseOwner("{not-json", OWNER_KEY), /DO owner record is invalid/); +}); + test("DO owner registry: keys encode owner scope and generation separately", () => { assert.equal(ownerKeyOf(OWNER_KEY), "do:owner:scope:do_0123456789abcdef0123456789abcdef%3ARoom%3Ashard0"); assert.equal( @@ -75,6 +101,43 @@ test("DO owner registry: keys encode owner scope and generation separately", () ); }); +test("DO owner registry: records stored under another owner scope fail closed", async () => { + const misplacedStorageId = "do_fedcba9876543210fedcba9876543210"; + const misplacedOwnerKey = `${misplacedStorageId}:Room:shard1`; + DO_OWNER_REGISTRY_TEST_STATE.store.set(ownerKeyOf(OWNER_KEY), JSON.stringify(ownerRecord({ + ownerKey: misplacedOwnerKey, + hostId: misplacedOwnerKey, + doStorageId: misplacedStorageId, + }))); + + await assert.rejects( + resolveDoOwner({ REDIS_ADDR: "redis:6379" }, invoke()), + /DO owner record is invalid/ + ); + assert.deepEqual(doOwnerRegistryWriteCommands(), []); +}); + +test("DO owner registry: records for another bundle scope fail closed before pointer lookup", async () => { + const misplacedPointerKey = "worker:do-storage:other:worker"; + DO_OWNER_REGISTRY_TEST_STATE.store.set(misplacedPointerKey, DO_STORAGE_ID); + DO_OWNER_REGISTRY_TEST_STATE.store.set(ownerKeyOf(OWNER_KEY), JSON.stringify(ownerRecord({ + ns: "other", + worker: "worker", + }))); + + await assert.rejects( + resolveDoOwner({ REDIS_ADDR: "redis:6379" }, invoke()), + /DO owner record is invalid/ + ); + assert.equal( + DO_OWNER_REGISTRY_TEST_STATE.redisState.commands.some((command) => ( + command[0] === "get" && command[1] === misplacedPointerKey + )), + false + ); + assert.deepEqual(doOwnerRegistryWriteCommands(), []); +}); + test("DO owner registry: lease guard config bounds values and permits test disable", () => { assert.equal(ownerLeaseGuardMs({}), 1000); assert.equal(ownerLeaseGuardMs({ DO_OWNER_LEASE_GUARD_MS: "0" }), 0); @@ -83,6 +146,7 @@ test("DO owner registry: lease guard config bounds values and permits test disab }); test("DO owner registry: claim writes owner generation counter", async () => { + setStoragePointer(); const owner = await resolveDoOwner({ REDIS_ADDR: "redis:6379" }, invoke()); assert.equal(owner.taskId, "task-a"); @@ -114,6 +178,7 @@ test("DO owner registry: corrupt generation counter fails closed", async () => { }); test("DO owner registry: claim retries WatchError before surfacing owner", async () => { + setStoragePointer(); DO_OWNER_REGISTRY_TEST_STATE.redisState.execFailures = 1; const owner = await resolveDoOwner({ REDIS_ADDR: "redis:6379" }, invoke()); @@ -121,12 +186,82 @@ test("DO owner registry: claim retries WatchError before surfacing owner", async assert.equal(owner.taskId, "task-a"); assert.equal(owner.generation, 1); assert.equal(DO_OWNER_REGISTRY_TEST_STATE.redisState.execFailures, 0); - assert.equal(DO_OWNER_REGISTRY_TEST_STATE.redisState.watchBatches.length, 2); + assert.equal( + DO_OWNER_REGISTRY_TEST_STATE.redisState.watchBatches.filter((keys) => + keys.includes(ownerKeyOf(OWNER_KEY)) + ).length, + 2 + ); +}); + +test("DO owner registry: first claim requires the active worker storage pointer", async () => { + await assert.rejects( + resolveDoOwner({ REDIS_ADDR: "redis:6379" }, invoke()), + (err) => { + assert.equal(/** @type {{ code?: unknown }} */ (err).code, "stale_owner_storage"); + return true; + } + ); + assert.deepEqual(doOwnerRegistryWriteCommands(), []); +}); + +test("DO owner registry: first claim refuses a worker under whole-delete lock", async () => { + setStoragePointer(); + DO_OWNER_REGISTRY_TEST_STATE.store.set(DELETE_LOCK_KEY, "whole:delete-token"); + + await assert.rejects( + resolveDoOwner({ REDIS_ADDR: "redis:6379" }, invoke()), + (err) => { + assert.equal(/** @type {{ code?: unknown }} */ (err).code, "stale_owner_storage"); + return true; + } + ); + assert.ok(DO_OWNER_REGISTRY_TEST_STATE.watchedKeys.includes(DELETE_LOCK_KEY)); + assert.deepEqual( + DO_OWNER_REGISTRY_TEST_STATE.redisState.commands.filter((command) => command[0] === "getManyWithTime"), + [["getManyWithTime", [ownerKeyOf(OWNER_KEY), DELETE_LOCK_KEY]]] + ); + assert.equal( + DO_OWNER_REGISTRY_TEST_STATE.redisState.commands.some((command) => ( + command[0] === "get" && command[1] === DELETE_LOCK_KEY + )), + false + ); + assert.deepEqual(doOwnerRegistryWriteCommands(), []); +}); + +test("DO owner registry: version delete lock does not interrupt active storage", async () => { + setStoragePointer(); + DO_OWNER_REGISTRY_TEST_STATE.store.set(DELETE_LOCK_KEY, "version:delete-token"); + + const owner = await resolveDoOwner({ REDIS_ADDR: "redis:6379" }, invoke()); + + assert.equal(owner.taskId, "task-a"); + assert.equal(owner.generation, 1); + assert.ok(DO_OWNER_REGISTRY_TEST_STATE.watchedKeys.includes(DELETE_LOCK_KEY)); +}); + +test("DO owner registry: delete lock appearing before claim commit aborts the claim", async () => { + setStoragePointer(); + DO_OWNER_REGISTRY_TEST_STATE.redisState.execFailures = 1; + DO_OWNER_REGISTRY_TEST_STATE.onExecFailure = () => { + DO_OWNER_REGISTRY_TEST_STATE.store.set(DELETE_LOCK_KEY, "whole:delete-token"); + }; + + await assert.rejects( + resolveDoOwner({ REDIS_ADDR: "redis:6379" }, invoke()), + (err) => { + assert.equal(/** @type {{ code?: unknown }} */ (err).code, "stale_owner_storage"); + return true; + } + ); + assert.equal(DO_OWNER_REGISTRY_TEST_STATE.store.has(ownerKeyOf(OWNER_KEY)), false); + assert.equal(DO_OWNER_REGISTRY_TEST_STATE.store.has(ownerGenerationKeyOf(OWNER_KEY)), false); }); test("DO owner registry: lost owner state cannot validate an old owner from another task", async () => { setStoragePointer(); - DO_OWNER_REGISTRY_TEST_STATE.taskIdentity = { taskId: "task-b", endpoint: "task-b:8788" }; + DO_OWNER_REGISTRY_TEST_STATE.taskIdentity = { taskId: "task-b", endpoint: "do-runtime-b:8788" }; const owner = await resolveDoOwner({ REDIS_ADDR: "redis:6379" }, invoke()); @@ -144,13 +279,13 @@ test("DO owner registry: lost owner state cannot validate an old owner from anot test("DO owner registry: same task reclaims generation one after owner state loss", async () => { setStoragePointer(); - DO_OWNER_REGISTRY_TEST_STATE.taskIdentity = { taskId: "task-b", endpoint: "task-b:8788" }; + DO_OWNER_REGISTRY_TEST_STATE.taskIdentity = { taskId: "task-b", endpoint: "do-runtime-b:8788" }; const owner = await resolveDoOwner({ REDIS_ADDR: "redis:6379" }, invoke()); assert.equal(owner.taskId, "task-b"); assert.equal(owner.generation, 1); - assert.equal(owner.endpoint, "task-b:8788"); + assert.equal(owner.endpoint, "do-runtime-b:8788"); assert.deepEqual(await assertCurrentOwner({ REDIS_ADDR: "redis:6379" }, owner), owner); }); @@ -199,7 +334,7 @@ test("DO owner registry: draining task returns a healthy remote owner", async () ownerKey, hostId: ownerKey, taskId: "task-b", - endpoint: "task-b:8788", + endpoint: "do-runtime-b:8788", }); setStoragePointer(); DO_OWNER_REGISTRY_TEST_STATE.store.set(ownerKeyOf(ownerKey), JSON.stringify(owner)); @@ -234,10 +369,11 @@ test("DO owner registry: expired takeover bumps generation monotonically", async ownerKey, hostId: ownerKey, taskId: "task-b", - endpoint: "task-b:8788", + endpoint: "do-runtime-b:8788", generation: 11, leaseExpiresAt: redisNow - EXPIRED_LEASE_AGE_MS, }); + setStoragePointer(); DO_OWNER_REGISTRY_TEST_STATE.store.set(ownerKeyOf(ownerKey), JSON.stringify(staleOwner)); DO_OWNER_REGISTRY_TEST_STATE.store.set(ownerGenerationKeyOf(ownerKey), "7"); @@ -481,7 +617,7 @@ test("DO owner registry: renewOwnedScopes forgets owners lost to another task", const remoteOwner = { ...owner, taskId: "task-b", - endpoint: "task-b:8788", + endpoint: "do-runtime-b:8788", generation: 9, }; DO_OWNER_REGISTRY_TEST_STATE.store.set(ownerKeyOf(ownerKey), JSON.stringify(remoteOwner)); @@ -510,7 +646,7 @@ test("DO owner registry: renewOwnedScopes logs lease loss with in-flight dispatc const remoteOwner = { ...owner, taskId: "task-b", - endpoint: "task-b:8788", + endpoint: "do-runtime-b:8788", generation: 9, }; DO_OWNER_REGISTRY_TEST_STATE.store.set(ownerKeyOf(ownerKey), JSON.stringify(remoteOwner)); diff --git a/tests/unit/do-runtime-actor.test.js b/tests/unit/do-runtime-actor.test.js index ca1fee4..5f441fa 100644 --- a/tests/unit/do-runtime-actor.test.js +++ b/tests/unit/do-runtime-actor.test.js @@ -5,9 +5,10 @@ import { loadDoHostActor, resetDoHostActorHarness, } from "../helpers/load-do-host-actor.js"; +import { assertJsonResponse } from "../helpers/response-json.js"; import { delay } from "../helpers/timing.js"; -const { WdlDoHostActor } = await loadDoHostActor(); +const { WdlDoHostActor, dispatchRpc } = await loadDoHostActor(); const harness = doHostActorHarnessState(); beforeEach(() => { @@ -39,6 +40,43 @@ function invoke(overrides = {}) { }; } +test("DO host actor: RPC dispatch passes request id through the internal wrapper boundary", async () => { + /** @type {Array<{ url: string, header: string | null, requestId: string | null, body: unknown }>} */ + const calls = []; + const facet = { + async fetch(/** @type {Request} */ request) { + const body = await request.json(); + calls.push({ + url: request.url, + header: request.headers.get("x-wdl-do-internal-rpc"), + requestId: request.headers.get("x-request-id"), + body, + }); + return Response.json({ ok: true, result: body }); + }, + }; + + const response = await dispatchRpc( + facet, + { method: "inspect", args: ["value"] }, + "rid-rpc" + ); + + await assertJsonResponse(response, 200, { + ok: true, + result: { + method: "inspect", + args: ["value"], + }, + }); + assert.deepEqual(calls, [{ + url: "https://do.internal/__wdl_rpc", + header: "1", + requestId: "rid-rpc", + body: { method: "inspect", args: ["value"] }, + }]); +}); + test("DO host actor: lease budget aborts a facet when the owner fence stops renewing", async () => { const host = actor({ DO_OWNER_LEASE_GUARD_MS: 0 }); const owner = { @@ -122,6 +160,41 @@ test("DO host actor: expired initial lease aborts before tenant dispatch", async assert.equal(harness.logs.at(-1).fields.reason, "expired"); }); +test("DO host actor: registry delay completes before the owner fence snapshot", async () => { + const host = actor({ DO_OWNER_LEASE_GUARD_MS: 0 }); + const registryStarted = Promise.withResolvers(); + const releaseRegistry = Promise.withResolvers(); + const owner = { + ownerKey: "do_0123456789abcdef0123456789abcdef:Room:shard0", + taskId: "task-a", + generation: 7, + leaseExpiresAt: Date.now() - 1, + }; + harness.assertResponses = [owner]; + harness.registryWait = releaseRegistry.promise; + harness.registryWaitStarted = () => registryStarted.resolve(undefined); + let ran = false; + + const dispatch = host.dispatchWithFence(invoke({ + doStorageId: "do_0123456789abcdef0123456789abcdef", + }), () => { + ran = true; + return new Response("should not run"); + }); + const first = await Promise.race([ + registryStarted.promise.then(() => "registry"), + dispatch.then(() => "dispatch-resolved", () => "dispatch-rejected"), + ]); + + assert.equal(first, "registry"); + assert.equal(harness.assertCalls, 0); + releaseRegistry.resolve(undefined); + await assert.rejects(dispatch, /owner lease has expired/); + + assert.equal(ran, false); + assert.equal(harness.assertCalls, 1); +}); + test("DO host actor: lease budget reschedules when renew extended the owner fence", async () => { const host = actor({ DO_OWNER_LEASE_GUARD_MS: 0 }); const owner = { @@ -231,3 +304,23 @@ test("DO host actor: registry remember failure is best-effort and does not fail assert.equal(harness.logs.at(-1).fields.member, "Room:alice"); assert.equal(harness.logs.at(-1).fields.worker_id, "demo:room:v1"); }); + +test("DO host actor strips tenant-supplied ownership error control markers", async () => { + const host = actor({ DO_OWNER_LEASE_GUARD_MS: 0 }); + harness.assertResponses = [{ + ownerKey: "do_0123456789abcdef0123456789abcdef:Room:shard0", + taskId: "task-a", + generation: 7, + leaseExpiresAt: Date.now() + 60_000, + }]; + + const response = await host.dispatchWithFence(invoke(), () => Promise.resolve( + new Response("tenant response", { + status: 503, + headers: { "x-wdl-do-ownership-error": "owner_fence_missing" }, + }) + )); + + assert.equal(await response.text(), "tenant response"); + assert.equal(response.headers.get("x-wdl-do-ownership-error"), null); +}); diff --git a/tests/unit/do-runtime-index.test.js b/tests/unit/do-runtime-index.test.js index ee10c55..d5a6b11 100644 --- a/tests/unit/do-runtime-index.test.js +++ b/tests/unit/do-runtime-index.test.js @@ -3,9 +3,11 @@ import { beforeEach, test } from "node:test"; import { doProtocolDataUrl } from "../helpers/load-do-protocol.js"; import { OBSERVABILITY_NOOP_URL } from "../helpers/mocks/observability.js"; +import { readJsonResponse } from "../helpers/response-json.js"; import { importSpecifierReplacements, moduleDataUrl, + readRepositoryJson, readRepositoryModuleSource, repositoryFileUrl, repositoryModuleDataUrl, @@ -29,8 +31,10 @@ function stub(src) { const protocolUrl = doProtocolDataUrl(); const sharedRespondUrl = repositoryFileUrl("shared/respond.js"); const sharedEnvUrl = repositoryFileUrl("shared/env.js"); +const sharedOptimisticRetryUrl = repositoryFileUrl("shared/optimistic-retry.js"); const sharedOwnerLeaseUrl = repositoryModuleDataUrl("shared/owner-lease.js", [ [/from "shared-env";/, `from ${JSON.stringify(sharedEnvUrl)};`], + [/from "shared-optimistic-retry";/, `from ${JSON.stringify(sharedOptimisticRetryUrl)};`], ]); const httpUrl = stub(readRepositoryModuleSource("do-runtime/http.js", importSpecifierReplacements({ "shared-respond": sharedRespondUrl, @@ -46,9 +50,11 @@ export function createHttpRequestScope({ request }) { } `); const workerIdUrl = repositoryFileUrl("shared/worker-id.js"); +const sharedVersionUrl = repositoryFileUrl("shared/version.js"); const actorUrl = stub(`export class WdlDoHostActor {}`); const alarmDispatchUrl = stub(readRepositoryModuleSource("do-runtime/alarm-dispatch.js", importSpecifierReplacements({ "shared-worker-id": workerIdUrl, + "shared-version": sharedVersionUrl, "do-runtime-protocol": protocolUrl, "do-runtime-http": httpUrl, }))); @@ -124,6 +130,7 @@ const IMPORT_STUBS = { "shared-owner-lease": sharedOwnerLeaseUrl, "shared-respond": sharedRespondUrl, "shared-request-scope": requestScopeUrl, + "shared-version": sharedVersionUrl, "shared-worker-id": workerIdUrl, "do-runtime-actor": actorUrl, "do-runtime-alarm-dispatch": alarmDispatchUrl, @@ -151,6 +158,7 @@ const src = readRepositoryModuleSource("do-runtime/index.js", importSpecifierRep const { default: app } = await import(stub(src)); const { DO_INVOKE_CONTENT_TYPE, DoRuntimeError, encodeDoInvokeRequest } = await import(protocolUrl); const doState = await import(stateUrl); +const versionFixture = readRepositoryJson("tests/fixtures/version-tags.json"); beforeEach(() => { /** @type {any} */ (globalThis).__doIndexHostResponse = null; @@ -250,6 +258,103 @@ test("do-runtime alarm dispatch endpoint invokes the local alarm shim path", asy assert.equal(fetchCall.input.url, "https://do-runtime.internal/invoke"); }); +test("do-runtime alarm dispatch delegates object identity validation", async () => { + const hostFetches = /** @type {any[]} */ (/** @type {any} */ (globalThis).__doIndexHostFetches); + const response = await app.fetch(internalRequest("https://do-runtime/internal/do/alarms/dispatch", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + ns: "tenant", + worker: "alarms", + version: "v7", + doStorageId: "do_0123456789abcdef0123456789abcdef", + className: "Room", + objectName: "\ud800", + retryCount: 0, + token: "row-token", + }), + }), env()); + + const body = await readJsonResponse(response, 400, "unpaired surrogate objectName"); + assert.equal(body.error, "invalid_request"); + assert.equal(hostFetches.length, 0); +}); + +test("do-runtime alarm dispatch requires retryCount and token before dispatch", async () => { + const hostFetches = /** @type {any[]} */ (/** @type {any} */ (globalThis).__doIndexHostFetches); + const validBody = { + ns: "tenant", + worker: "alarms", + version: "v7", + doStorageId: "do_0123456789abcdef0123456789abcdef", + className: "Room", + objectName: "alice", + retryCount: 0, + token: "row-token", + }; + for (const { label, field, value } of [ + { label: "missing retryCount", field: "retryCount", value: undefined }, + { label: "null retryCount", field: "retryCount", value: null }, + { label: "missing token", field: "token", value: undefined }, + { label: "null token", field: "token", value: null }, + ]) { + const response = await app.fetch(internalRequest("https://do-runtime/internal/do/alarms/dispatch", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ ...validBody, [field]: value }), + }), env()); + + const body = await readJsonResponse(response, 400, label); + assert.equal(body.error, "invalid_request", label); + assert.equal(hostFetches.length, 0, label); + } +}); + +test("do-runtime alarm dispatch delegates retryCount validation before dispatch", async () => { + const hostFetches = /** @type {any[]} */ (/** @type {any} */ (globalThis).__doIndexHostFetches); + const response = await app.fetch(internalRequest("https://do-runtime/internal/do/alarms/dispatch", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + ns: "tenant", + worker: "alarms", + version: "v7", + doStorageId: "do_0123456789abcdef0123456789abcdef", + className: "Room", + objectName: "alice", + retryCount: "2", + token: "row-token", + }), + }), env()); + + const body = await readJsonResponse(response, 400, "non-number retryCount"); + assert.equal(body.error, "invalid_request"); + assert.equal(hostFetches.length, 0); +}); + +test("do-runtime alarm dispatch rejects non-canonical versions from the shared fixture", async () => { + for (const { tag, parsed } of versionFixture.cases) { + if (parsed != null) continue; + const response = await app.fetch(internalRequest("https://do-runtime/internal/do/alarms/dispatch", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + ns: "tenant", + worker: "alarms", + version: tag, + doStorageId: "do_0123456789abcdef0123456789abcdef", + className: "Room", + objectName: "alice", + retryCount: 0, + token: "row-token", + }), + }), env()); + assert.equal(response.status, 400, tag); + assert.match((await jsonBody(response)).message, /version/, tag); + } + assert.deepEqual(/** @type {any[]} */ (/** @type {any} */ (globalThis).__doIndexHostFetches), []); +}); + test("do-runtime alarm dispatch endpoint maps failed local dispatch to retryable 503", async () => { /** @type {any} */ (globalThis).__doIndexHostResponse = Response.json({ error: "alarm_failed", @@ -488,6 +593,26 @@ test("do-runtime storage-delete-worker classifies invalid members as 207 partial }); }); +test("do-runtime storage-delete-worker rejects non-canonical versions before dispatch", async () => { + const response = await app.fetch(internalRequest("https://do-runtime/internal/do/storage/delete-worker", { + method: "POST", + body: JSON.stringify({ + ns: "tenant", + worker: "chat", + version: "v9007199254740992", + doStorageId: "do_0123456789abcdef0123456789abcdef", + members: ["Room:room-a:0"], + }), + }), env()); + + assert.equal(response.status, 400); + assert.deepEqual(await jsonBody(response), { + error: "invalid_request", + message: "version is invalid", + }); + assert.deepEqual(/** @type {any[]} */ (/** @type {any} */ (globalThis).__doIndexHostFetches), []); +}); + test("do-runtime storage-delete-worker preserves stable member error codes from host responses", async () => { /** @type {any} */ (globalThis).__doIndexHostResponse = Response.json({ error: "stale_owner_generation", diff --git a/tests/unit/do-runtime-load.test.js b/tests/unit/do-runtime-load.test.js index db630f4..e95965c 100644 --- a/tests/unit/do-runtime-load.test.js +++ b/tests/unit/do-runtime-load.test.js @@ -11,6 +11,10 @@ import { installMockFetch, withMockedFetch } from "../helpers/mock-fetch.js"; import { withMockedGlobal } from "../helpers/mock-global.js"; import { installConsoleMethodCapture } from "../helpers/output-capture.js"; import { sharedInternalAuthUrl } from "../helpers/runtime-proxy-stub.js"; +import { + DO_RUNTIME_RESERVED_MODULE, + doRuntimeInjectedModuleSources, +} from "../../do-runtime/load-code-budget.js"; /** @type {any} */ (globalThis).__doLoadFetches = []; /** @type {any} */ (globalThis).__doLoadResponses = []; @@ -221,7 +225,9 @@ test("DO runtime load: abort timeout is retried inside the load budget", async ( test("DO runtime load: applies the host-binding wrapper before the alarm wrapper", async () => { /** @type {any} */ (globalThis).__doRuntimeHostWrap = (/** @type {any} */ workerCode) => { - workerCode.modules["_wdl-wrapper.js"] = "export class Room {}; export default {};"; + workerCode.modules["_wdl-wrapper.js"] = [ + "export class Room {}; export default {};", + ].join("\n"); workerCode.mainModule = "_wdl-wrapper.js"; }; /** @type {any} */ (globalThis).__doLoadResponses.push(jsonLoadResponse(200, { @@ -248,17 +254,54 @@ test("DO runtime load: applies the host-binding wrapper before the alarm wrapper assert.equal(loaded.mainModule, "_wdl-do-runtime-wrapper.js"); assert.deepEqual(loaded.compatibilityFlags, ["nodejs_compat", "delete_all_preserves_alarm"]); - assert.equal(loaded.modules["_wdl-wrapper.js"], "export class Room {}; export default {};"); + assert.doesNotMatch(loaded.modules["_wdl-wrapper.js"], /__wdlRunWithRequestContext/); assert.match(loaded.modules["_wdl-do-alarm-shim.js"], /function withoutInternalEnv/); assert.match(loaded.modules["_wdl-do-alarm-shim.js"], /function deleteAllKvStorage/); assert.match(loaded.modules["_wdl-do-alarm-shim.js"], /function deleteAllSqlStorage/); assert.match(loaded.modules["_wdl-do-alarm-shim.js"], /export function wrapDurableObjectClass/); assert.match(loaded.modules["_wdl-do-alarm-shim.js"], /delete out\[ALARMS_BINDING\];/); + assert.match(loaded.modules["_wdl-do-alarm-shim.js"], /const objectDefineProperty = Object\.defineProperty;/); + assert.match(loaded.modules["_wdl-do-alarm-shim.js"], /return new NativeProxy\(storage,/); assert.doesNotMatch(loaded.modules["_wdl-do-alarm-shim.js"], /delete out\.__WDL_HOST_BINDINGS_WRAPPED__/); const wrapper = loaded.modules["_wdl-do-runtime-wrapper.js"]; - assert.match(wrapper, /import \* as user from "\.\/_wdl-wrapper\.js";/); - assert.match(wrapper, /import \{ wrapDurableObjectClass \} from "\.\/_wdl-do-alarm-shim\.js";/); - assert.match(wrapper, /export class Room extends wrapDurableObjectClass\(user\.Room, "Room"\)/); + assert.match(wrapper, /import \* as __WdlUserModule__ from "\.\/_wdl-wrapper\.js";/); + assert.match(wrapper, /import \{ wrapDurableObjectClass as __WdlWrapDurableObjectClass__ \} from "\.\/_wdl-do-alarm-shim\.js";/); + assert.ok( + wrapper.indexOf('from "./_wdl-do-alarm-shim.js";') < + wrapper.indexOf('import * as __WdlUserModule__ from "./_wdl-wrapper.js";') + ); + assert.match(wrapper, /__WdlUserModule__\.Room,\s+"Room"/); + assert.doesNotMatch(wrapper, /__wdlRunWithRequestContext/); +}); + +test("DO runtime wrapper aliases legal class names without declaration collisions", async () => { + const classNames = ["user", "wrapDurableObjectClass"]; + const injections = Object.fromEntries(doRuntimeInjectedModuleSources("worker.js", { + bindings: { + USER: { type: "do", className: classNames[0] }, + WRAPPER: { type: "do", className: classNames[1] }, + }, + }, "export function wrapDurableObjectClass() {}")); + const userUrl = moduleDataUrl(` + export class user {} + export class wrapDurableObjectClass {} + `); + const shimUrl = moduleDataUrl(` + export function wrapDurableObjectClass(Base) { + return class extends Base {}; + } + `); + const source = applyModuleReplacements(injections[DO_RUNTIME_RESERVED_MODULE], [ + ['from "./_wdl-do-alarm-shim.js"', `from ${JSON.stringify(shimUrl)}`], + [/from "\.\/worker\.js"/g, `from ${JSON.stringify(userUrl)}`], + ]); + const wrapped = await import(moduleDataUrl(source)); + const userModule = await import(userUrl); + + for (const name of classNames) { + assert.equal(wrapped[name].name, name); + assert.ok(wrapped[name].prototype instanceof userModule[name]); + } }); test("DO runtime load: rejects reserved alarm shim module collisions", async () => { diff --git a/tests/unit/do-runtime-protocol.test.js b/tests/unit/do-runtime-protocol.test.js index 60790cf..ebc7b29 100644 --- a/tests/unit/do-runtime-protocol.test.js +++ b/tests/unit/do-runtime-protocol.test.js @@ -3,17 +3,33 @@ import { test } from "node:test"; import { decodeDoEnvelope } from "../helpers/do-envelope.js"; import { loadDoProtocol } from "../helpers/load-do-protocol.js"; +import { readRepositoryJson } from "../helpers/load-shared-module.js"; import { assertJsonResponse } from "../helpers/response-json.js"; +import { + dispatchDoInvokeWithHintCache, + retryableOwnerRaceResponse, + staleDoOwnerHintResponse, +} from "../../runtime/_wdl-do-transport.js"; +import { DO_HOST_SHARD_COUNT, MAX_ID_BYTES } from "../../do-runtime/protocol/wire-grammar.js"; +import { + doOwnerHintHeaders, + doOwnershipErrorHeaders, +} from "../helpers/do-owner-hint.js"; +import { withMockedProperty } from "../helpers/mock-global.js"; const { DoRuntimeError, DO_INVOKE_CONTENT_TYPE, + DO_OWNERSHIP_ERROR_CONTROL_HEADER, + DO_OWNERSHIP_CODE, buildFacetName, buildAlarmRequest, buildForwardRequest, + buildRpcRequest, encodeDoInvokeRequest, buildLocalActorRequest, doErrorResponse, + doPlatformErrorResponse, hostIdForObject, hostIdForShard, normalizeDoConnectRequest, @@ -22,6 +38,82 @@ const { readLocalActorInvokeRequest, readJsonBody, } = await loadDoProtocol(); +const versionFixture = readRepositoryJson("tests/fixtures/version-tags.json"); +const doAlarmIdentityFixture = /** @type {any} */ ( + readRepositoryJson("tests/fixtures/do-alarm-identity.json") +); + +test("DO ownership errors stay aligned with runtime hint and retry handling", () => { + const ownershipCodes = new Set(Object.values(DO_OWNERSHIP_CODE)); + /** @type {Set} */ + const ownerRaceRetryCodes = new Set([ + DO_OWNERSHIP_CODE.STALE_OWNER_GENERATION, + DO_OWNERSHIP_CODE.OWNER_CLAIM_RACED, + DO_OWNERSHIP_CODE.OWNER_FENCE_MISSING, + DO_OWNERSHIP_CODE.OWNER_LEASE_EXPIRED, + DO_OWNERSHIP_CODE.OWNER_LEASE_TOO_SHORT, + ]); + for (const code of ownershipCodes) { + const untrusted = Response.json({ error: code }, { status: 503 }); + assert.equal(staleDoOwnerHintResponse(untrusted), false, code); + assert.equal(retryableOwnerRaceResponse(untrusted), false, code); + + const response = Response.json({ error: code }, { + status: 503, + headers: doOwnershipErrorHeaders(code), + }); + assert.equal(staleDoOwnerHintResponse(response), true, code); + assert.equal( + retryableOwnerRaceResponse(response), + ownerRaceRetryCodes.has(code), + code + ); + } +}); + +test("DO invoke dispatch does not cache hints carried by owner-race responses", async () => { + /** @type {Array<{ url: string, init: RequestInit | undefined }>} */ + const routerCalls = []; + /** @type {unknown[]} */ + const cachedHints = []; + const init = { + method: "POST", + headers: { "x-wdl-do-accept-owner-hint": "1" }, + }; + + const response = await dispatchDoInvokeWithHintCache({ + async routerFetch(url, requestInit) { + routerCalls.push({ url, init: requestInit }); + return routerCalls.length === 1 + ? Response.json({ error: "stale_owner_generation", message: "owner moved" }, { + status: 503, + headers: doOwnershipErrorHeaders("stale_owner_generation", doOwnerHintHeaders()), + }) + : new Response("retried"); + }, + routerUrl: "http://do-runtime/internal/do/invoke", + async ownerFetch() { + assert.fail("owner endpoint must not receive an owner-race response hint"); + }, + ownerPath: "/internal/do/invoke", + init, + cache: { + get() { + return null; + }, + set(_key, hint) { + cachedHints.push(hint); + }, + delete() {}, + }, + hintKey: "room-a", + }); + + assert.equal(await response.text(), "retried"); + assert.equal(routerCalls.length, 2); + assert.deepEqual(cachedHints, []); + assert.equal(new Headers(routerCalls[1].init?.headers).get("x-wdl-do-accept-owner-hint"), null); +}); const DO_STORAGE_ID = "do_0123456789abcdef0123456789abcdef"; const CHAT_ROOM_HOST_ID = `${DO_STORAGE_ID}:ChatRoom:shard12`; @@ -56,19 +148,6 @@ function withRequestBody(invoke, bodyBytes) { }; } -const INLINE_BODY = { - ...BASE_BODY, - hostId: CHAT_ROOM_HOST_ID, - workerId: "tenant:chat:1", - workerCode: { - compatibilityDate: "2026-04-24", - mainModule: "worker.js", - modules: { - "worker.js": "export class ChatRoom {}", - }, - }, -}; - test("normalizes do invoke request", () => { const invoke = normalizeDoInvokeRequest(BASE_BODY); @@ -89,20 +168,48 @@ test("normalizes do invoke request", () => { assert.equal(buildFacetName(invoke), "ChatRoom:room-a"); }); -test("normalizes do invoke request for reserved namespaces", () => { - const invoke = normalizeDoInvokeRequest({ - ...BASE_BODY, - ns: "__system__", - }); - - assert.equal(invoke.workerId, "__system__:chat:v1"); - assert.deepEqual(invoke.props, { - ns: "__system__", - worker: "chat", - version: "v1", - doStorageId: DO_STORAGE_ID, - className: "ChatRoom", - }); +test("DO invoke version ingress matches the shared JS/Rust fixture", () => { + for (const { tag, parsed } of versionFixture.cases) { + if (parsed == null) { + assert.throws( + () => normalizeDoInvokeRequest({ ...BASE_BODY, version: tag }), + /version/, + tag + ); + } else { + const invoke = normalizeDoInvokeRequest({ ...BASE_BODY, version: tag }); + assert.ok("version" in invoke, tag); + assert.equal(invoke.version, tag, tag); + } + } +}); + +test("normalizes do invoke requests for runtime-load reserved namespaces", () => { + for (const ns of ["__system__", "__platform__"]) { + const invoke = normalizeDoInvokeRequest({ + ...BASE_BODY, + ns, + }); + + assert.equal(invoke.workerId, `${ns}:chat:v1`); + assert.deepEqual(invoke.props, { + ns, + worker: "chat", + version: "v1", + doStorageId: DO_STORAGE_ID, + className: "ChatRoom", + }); + } +}); + +test("DO invoke namespace ingress uses the canonical runtime-load grammar", () => { + for (const ns of ["-tenant", "tenant-", "a".repeat(64), "admin", "__community__"]) { + assert.throws( + () => normalizeDoInvokeRequest({ ...BASE_BODY, ns }), + /ns is not valid/, + ns + ); + } }); test("normalizes do invoke request for mixed-case worker names", () => { @@ -153,9 +260,10 @@ test("normalizes do alarm invoke request", async () => { assert.deepEqual(invoke.alarm, { retryCount: 2, isRetry: true, token: "alarm-token" }); assert.equal("request" in invoke ? invoke.request : undefined, undefined); - const request = buildAlarmRequest(invoke.alarm); + const request = buildAlarmRequest(invoke.alarm, "rid-alarm"); assert.equal(request.method, "POST"); assert.equal(request.headers.get("x-wdl-do-internal-alarm"), "1"); + assert.equal(request.headers.get("x-request-id"), "rid-alarm"); assert.deepEqual(await request.json(), { retryCount: 2, isRetry: true, token: "alarm-token" }); }); @@ -178,6 +286,22 @@ test("normalizes do rpc invoke request", () => { }); }); +test("builds private rpc dispatch requests", async () => { + const request = buildRpcRequest({ + method: "addMessage", + args: ["hello", { role: "user" }], + }, "rid-rpc"); + + assert.equal(request.method, "POST"); + assert.equal(request.url, "https://do.internal/__wdl_rpc"); + assert.equal(request.headers.get("x-wdl-do-internal-rpc"), "1"); + assert.equal(request.headers.get("x-request-id"), "rid-rpc"); + assert.deepEqual(await request.json(), { + method: "addMessage", + args: ["hello", { role: "user" }], + }); +}); + test("rejects invalid do rpc shapes", () => { assert.throws( () => normalizeDoInvokeRequest({ @@ -229,29 +353,39 @@ test("rejects invalid do rpc shapes", () => { ); }); -test("normalizes inline workerCode only for test hooks", () => { - assert.throws( - () => normalizeDoInvokeRequest(INLINE_BODY), - /workerCode is only accepted when DO_TEST_HOOKS=1/ - ); - const invoke = normalizeDoInvokeRequest(INLINE_BODY, { allowInlineWorkerCode: true }); - assert.equal(invoke.hostId, CHAT_ROOM_HOST_ID); - assert.equal("workerCode" in invoke ? "allowExperimental" in invoke.workerCode : undefined, false); +test("do-runtime RPC JSON validation uses captured intrinsics", async () => { + await withMockedProperty(Object, "entries", () => [], () => { + assert.throws( + () => normalizeDoInvokeRequest({ + ...BASE_BODY, + kind: "rpc", + rpc: { method: "addMessage", args: [{ fn() {} }] }, + }), + (error) => error instanceof Error && error.message === "rpc.args[0].fn must be JSON data" + ); + }); + await withMockedProperty(Number, "isFinite", () => true, () => { + assert.throws( + () => normalizeDoInvokeRequest({ + ...BASE_BODY, + kind: "rpc", + rpc: { method: "addMessage", args: [Number.NaN] }, + }), + (error) => error instanceof Error && error.message === "rpc.args[0] must be a finite number" + ); + }); }); -test("rejects experimental workerd compatibility flags in inline workerCode", () => { +test("rejects inline workerCode", () => { assert.throws( () => normalizeDoInvokeRequest({ - ...INLINE_BODY, + ...BASE_BODY, workerCode: { - ...INLINE_BODY.workerCode, - compatibilityFlags: ["nodejs_compat", "unsafe_module"], + mainModule: "worker.js", + modules: { "worker.js": "export class ChatRoom {}" }, }, - }, { allowInlineWorkerCode: true }), - (err) => err instanceof DoRuntimeError && - err.status === 400 && - err.code === "experimental_compat_flag_unsupported" && - /"unsafe_module"/.test(err.message) + }), + /workerCode is not accepted/ ); }); @@ -263,6 +397,7 @@ test("builds forwarded Request for user durable object fetch", async () => { headers: { ...BASE_BODY.request.headers, "x-wdl-do-internal-alarm": "1", + "x-wdl-do-internal-rpc": "1", "x-wdl-internal-auth": "platform-token", }, }, @@ -274,6 +409,7 @@ test("builds forwarded Request for user durable object fetch", async () => { assert.equal(request.url, "https://demo.workers.example/messages"); assert.equal(request.headers.get("content-type"), "text/plain"); assert.equal(request.headers.get("x-wdl-do-internal-alarm"), null); + assert.equal(request.headers.get("x-wdl-do-internal-rpc"), null); assert.equal(request.headers.get("x-wdl-internal-auth"), null); assert.equal(await request.text(), "hello"); }); @@ -290,6 +426,7 @@ test("normalizes DO connect without exposing internal auth to tenant code", () = "x-wdl-do-object-name": "room-a", "x-wdl-do-request-url": "https://demo.workers.example/ws", "x-wdl-internal-auth": "platform-token", + "x-wdl-do-ownership-error": "stale_owner_generation", "x-tenant-visible": "ok", }, })); @@ -407,6 +544,7 @@ test("normalizes do websocket connect request from internal headers", () => { "x-wdl-do-owner-endpoint": "do-runtime-a:8788", "x-wdl-do-owner-generation": "4", "x-wdl-do-owner-hint": "1", + "x-wdl-do-ownership-error": "stale_owner_generation", }, }); const invoke = normalizeDoConnectRequest(request); @@ -426,29 +564,35 @@ test("normalizes do websocket connect request from internal headers", () => { assert.equal(invoke.request.headers.some(([name]) => name === "x-wdl-do-owner-generation"), false); assert.equal(invoke.request.headers.some(([name]) => name === "x-wdl-do-owner-endpoint"), false); assert.equal(invoke.request.headers.some(([name]) => name === "x-wdl-do-owner-hint"), false); + assert.equal(invoke.request.headers.some(([name]) => name === "x-wdl-do-ownership-error"), false); assert.ok(invoke.request.headers.some(([name, value]) => name === "upgrade" && value === "websocket")); assert.ok(invoke.request.headers.some(([name, value]) => name === "x-request-id" && value === "rid-1")); }); test("rejects invalid do alarm retry counts", () => { - assert.throws( - () => normalizeDoInvokeRequest({ - ...BASE_BODY, - kind: "alarm", - alarm: { retryCount: -1 }, - }), - /alarm\.retryCount must be a non-negative integer/ - ); + for (const retryCount of [-1, 1.5, "2", true, false, "", [], [2]]) { + assert.throws( + () => normalizeDoInvokeRequest({ + ...BASE_BODY, + kind: "alarm", + alarm: { retryCount }, + }), + /alarm\.retryCount must be a non-negative integer/, + `retryCount=${JSON.stringify(retryCount)}` + ); + } }); test("rejects invalid owner fence generation", () => { - assert.throws( - () => normalizeDoInvokeRequest({ - ...BASE_BODY, - owner: { ownerKey: CHAT_ROOM_HOST_ID, taskId: "task-a", generation: -1 }, - }), - /owner\.generation must be a non-negative integer/ - ); + for (const generation of [-1, 0, 9007199254740992]) { + assert.throws( + () => normalizeDoInvokeRequest({ + ...BASE_BODY, + owner: { ownerKey: CHAT_ROOM_HOST_ID, taskId: "task-a", generation }, + }), + /owner\.generation must be a positive safe integer/ + ); + } }); test("rejects invalid class names", () => { @@ -462,6 +606,37 @@ test("rejects invalid class names", () => { ); }); +/** @param {string | { repeat: string, count: number }} value */ +function fixtureString(value) { + return typeof value === "string" ? value : value.repeat.repeat(value.count); +} + +test("DO alarm ingress matches the cross-language identity fixture", () => { + assert.equal(doAlarmIdentityFixture.maxBytes, MAX_ID_BYTES); + assert.equal(doAlarmIdentityFixture.hostShardCount, DO_HOST_SHARD_COUNT); + for (const entry of doAlarmIdentityFixture.cases) { + const input = { + ...BASE_BODY, + kind: "alarm", + doStorageId: fixtureString(entry.doStorageId), + className: fixtureString(entry.className), + objectName: fixtureString(entry.objectName), + alarm: { retryCount: 0, token: fixtureString(entry.token) }, + }; + if (entry.valid) { + assert.doesNotThrow(() => normalizeDoInvokeRequest(input), entry.name); + } else { + assert.throws( + () => normalizeDoInvokeRequest(input), + (err) => err instanceof DoRuntimeError && + err.status === 400 && + err.code === "invalid_request", + entry.name + ); + } + } +}); + test("rejects bare numeric versions", () => { assert.throws( () => normalizeDoInvokeRequest({ ...BASE_BODY, version: "1" }), @@ -469,36 +644,25 @@ test("rejects bare numeric versions", () => { ); }); -test("rejects missing main module", () => { +test("rejects malformed, out-of-range, and mismatched host ids", () => { assert.throws( - () => normalizeDoInvokeRequest({ - ...BASE_BODY, - hostId: CHAT_ROOM_HOST_ID, - workerId: "tenant:chat:1", - workerCode: { - mainModule: "missing.js", - modules: { "worker.js": "export default {};" }, - }, - }, { allowInlineWorkerCode: true }), - /workerCode\.mainModule must reference a module/ - ); -}); - -test("rejects malformed inline test-hook host ids", () => { - assert.throws( - () => normalizeDoInvokeRequest({ - ...INLINE_BODY, - hostId: "tenant-worker", - }, { allowInlineWorkerCode: true }), + () => normalizeDoInvokeRequest({ ...BASE_BODY, hostId: "tenant-worker" }), /hostId is not valid/ ); assert.throws( () => normalizeDoInvokeRequest({ - ...INLINE_BODY, + ...BASE_BODY, hostId: `${DO_STORAGE_ID}:ChatRoom:shard16`, - }, { allowInlineWorkerCode: true }), + }), /hostId shard is not valid/ ); + assert.throws( + () => normalizeDoInvokeRequest({ + ...BASE_BODY, + hostId: `${DO_STORAGE_ID}:ChatRoom:shard012`, + }), + /hostId is not valid/ + ); assert.throws( () => normalizeDoInvokeRequest({ ...BASE_BODY, @@ -508,6 +672,15 @@ test("rejects malformed inline test-hook host ids", () => { ); }); +test("generated host ids honor the aggregate protocol size limit", () => { + const maxHostId = hostIdForShard("a".repeat(496), "ChatRoom", 0); + assert.equal(Buffer.byteLength(maxHostId), 512); + assert.throws( + () => hostIdForShard("a".repeat(497), "ChatRoom", 0), + /hostId is too large/ + ); +}); + test("shards object names using UTF-8 bytes", () => { assert.equal(hostIdForObject(DO_STORAGE_ID, "ChatRoom", "中文"), `${DO_STORAGE_ID}:ChatRoom:shard5`); assert.equal(hostIdForObject(DO_STORAGE_ID, "ChatRoom", "🚀"), `${DO_STORAGE_ID}:ChatRoom:shard10`); @@ -625,27 +798,17 @@ test("rejects invalid header names and control characters", () => { ); }); -test("rejects inline workerCode module limits", () => { - const tooManyModules = Object.fromEntries( - Array.from({ length: 129 }, (_, index) => [`module-${index}.js`, "export default {};"]) - ); - assert.throws( - () => normalizeDoInvokeRequest({ - ...INLINE_BODY, - workerCode: { ...INLINE_BODY.workerCode, modules: tooManyModules }, - }, { allowInlineWorkerCode: true }), - /workerCode\.modules has too many modules/ - ); - assert.throws( - () => normalizeDoInvokeRequest({ - ...INLINE_BODY, - workerCode: { - ...INLINE_BODY.workerCode, - modules: { "worker.js": "x".repeat(1024 * 1024 + 1) }, - }, - }, { allowInlineWorkerCode: true }), - /workerCode\.modules\.worker\.js is too large/ - ); +test("rejects unpaired surrogates in DO object identities", () => { + for (const objectName of ["\ud800", "\udc00"]) { + assert.throws( + () => normalizeDoInvokeRequest({ ...BASE_BODY, objectName }), + /objectName must contain well-formed Unicode/, + ); + assert.throws( + () => hostIdForObject(BASE_BODY.doStorageId, BASE_BODY.className, objectName), + /objectName must contain well-formed Unicode/, + ); + } }); test("maps invalid JSON bodies to protocol errors", async () => { @@ -675,6 +838,38 @@ test("do runtime error responses keep internal exception messages out of the wir }); }); +test("do runtime ownership errors carry a private control marker", async () => { + const ownershipResponse = doPlatformErrorResponse(new DoRuntimeError( + 503, + DO_OWNERSHIP_CODE.OWNER_FENCE_MISSING, + "DO owner fence is missing" + )); + assert.equal( + ownershipResponse.headers.get(DO_OWNERSHIP_ERROR_CONTROL_HEADER), + DO_OWNERSHIP_CODE.OWNER_FENCE_MISSING + ); + + const tenantResponse = doPlatformErrorResponse(new DoRuntimeError( + 503, + "tenant_failure", + "Tenant failure" + )); + assert.equal( + tenantResponse.headers.get(DO_OWNERSHIP_ERROR_CONTROL_HEADER), + null + ); + + const forgedResponse = doPlatformErrorResponse({ + status: 503, + code: DO_OWNERSHIP_CODE.OWNER_FENCE_MISSING, + message: "forged ownership error", + }); + assert.equal( + forgedResponse.headers.get(DO_OWNERSHIP_ERROR_CONTROL_HEADER), + null + ); +}); + test("do runtime error responses keep top-level fields stable while preserving nested details", async () => { const response = doErrorResponse(new DoRuntimeError(503, "owner_unavailable", "DO owner is unavailable", { error: "socket leaked", diff --git a/tests/unit/do-task-identity.test.js b/tests/unit/do-task-identity.test.js index 716ba57..abfca0d 100644 --- a/tests/unit/do-task-identity.test.js +++ b/tests/unit/do-task-identity.test.js @@ -5,17 +5,23 @@ import { applyModuleReplacements, moduleDataUrl, readRepositoryFile, + repositoryFileUrl, sharedModuleDataUrl, } from "../helpers/load-shared-module.js"; import { doProtocolDataUrl } from "../helpers/load-do-protocol.js"; const PROTOCOL_URL = doProtocolDataUrl(); const SHARED_TASK_IDENTITY_URL = sharedModuleDataUrl("shared/task-identity.js"); +const SHARED_OWNER_ENDPOINT_URL = repositoryFileUrl("shared/owner-endpoint.js"); const src = applyModuleReplacements(readRepositoryFile("do-runtime/task-identity.js"), [ [ /import \{ createTaskIdentityResolver \} from "shared-task-identity";/, `import { createTaskIdentityResolver } from ${JSON.stringify(SHARED_TASK_IDENTITY_URL)};`, ], + [ + /import \{ validOwnerEndpointForService \} from "shared-owner-endpoint";/, + `import { validOwnerEndpointForService } from ${JSON.stringify(SHARED_OWNER_ENDPOINT_URL)};`, + ], [ /import \{ DoRuntimeError \} from "do-runtime-protocol";/, `import { DoRuntimeError } from ${JSON.stringify(PROTOCOL_URL)};`, @@ -65,6 +71,13 @@ test("DO task identity: env task id and endpoint win", async () => { }); }); +test("DO task identity: public owner endpoints are rejected", () => { + assert.throws( + () => taskIdentityFromEnv({ DO_TASK_ID: "task-a", DO_TASK_ENDPOINT: "8.8.8.8:8788" }), + (err) => isRuntimeError(err, 503, "task_identity_unavailable") + ); +}); + test("DO task identity: partial env identity is rejected", () => { assert.throws( () => taskIdentityFromEnv({ DO_TASK_ID: "task-only" }), @@ -81,11 +94,11 @@ test("DO task identity: ECS metadata yields task arn and private IPv4 endpoint", Networks: [{ IPv4Addresses: ["10.0.42.17"] }], }, ], - }, { DO_TASK_PORT: "9798" }); + }); assert.deepEqual(identity, { taskId: "arn:aws:ecs:us-east-1:123456789012:task/cluster/abc", - endpoint: "10.0.42.17:9798", + endpoint: "10.0.42.17:8788", source: "ecs-metadata", }); }); diff --git a/tests/unit/fake-redis.test.js b/tests/unit/fake-redis.test.js index ecdb171..2847117 100644 --- a/tests/unit/fake-redis.test.js +++ b/tests/unit/fake-redis.test.js @@ -1,7 +1,21 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { redisConformanceCases } from "../helpers/redis-conformance-cases.js"; -import { createFakeRedis, FakeRedisWatchError } from "../helpers/mocks/fake-redis.js"; +import { + createFakeRedis, + FakeRedisWatchError, + sharedRedisStubUrl, +} from "../helpers/mocks/fake-redis.js"; + +test("shared Redis test modules use the fake WatchError and production bulk decoder", async () => { + const { WatchError, decodeBulk } = await import(sharedRedisStubUrl()); + assert.equal(WatchError, FakeRedisWatchError); + assert.equal(decodeBulk(undefined), undefined); + assert.equal(decodeBulk(null), null); + assert.equal(decodeBulk("value"), "value"); + assert.equal(decodeBulk(new TextEncoder().encode("bytes")), "bytes"); + assert.equal(decodeBulk(42), "42"); +}); /** * @param {ReturnType} redis @@ -94,6 +108,16 @@ test("fake redis session records single and batched hash reads", async () => { ]); }); +test("fake redis getManyWithTime matches the production empty-input contract", async () => { + const redis = createFakeRedis(); + await redis.session(async (session) => { + await assert.rejects( + session.getManyWithTime([]), + /getManyWithTime requires at least one key/ + ); + }); +}); + test("fake redis supports hash existence and key reads on clients and sessions", async () => { const redis = createFakeRedis(); redis.hashes.set("hash:a", { one: "1", two: "2" }); diff --git a/tests/unit/gateway-lib.test.js b/tests/unit/gateway-lib.test.js index 7a588e2..e2e1f95 100644 --- a/tests/unit/gateway-lib.test.js +++ b/tests/unit/gateway-lib.test.js @@ -1,6 +1,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { + deleteGatewayInternalHeaders, escapeRegex, classifyHost, isCanonicalPatternHost, @@ -11,6 +12,24 @@ import { } from "../../gateway/lib.js"; import { NS_PATTERN, isValidRouteNs } from "../../shared/ns-pattern.js"; +test("deleteGatewayInternalHeaders removes fixed and prefixed private headers", () => { + const headers = new Headers({ + "x-worker-id": "tenant:worker:v1", + "x-worker-prefix": "/worker", + "x-wdl-upstream-binding": "RUNTIME_USER", + "x-wdl-future-private": "private", + "x-request-id": "keep-request-id", + "sec-websocket-protocol": "keep-protocol", + }); + + deleteGatewayInternalHeaders(headers); + + assert.deepEqual(Object.fromEntries(headers), { + "sec-websocket-protocol": "keep-protocol", + "x-request-id": "keep-request-id", + }); +}); + /** * @param {any} sorted * @param {string} pathname diff --git a/tests/unit/gateway-websocket.test.js b/tests/unit/gateway-websocket.test.js index 0de506a..b2d55e9 100644 --- a/tests/unit/gateway-websocket.test.js +++ b/tests/unit/gateway-websocket.test.js @@ -77,6 +77,7 @@ const { proxyGatewayWebSocket, webSocketProxyOptionsFromEnv } = await importRepo [ [/from "shared-observability";/, `from ${JSON.stringify(repositoryFileUrl("shared/observability.js"))};`], [/from "shared-respond";/, `from ${JSON.stringify(repositoryFileUrl("shared/respond.js"))};`], + [/from "gateway-lib";/, `from ${JSON.stringify(repositoryFileUrl("gateway/lib.js"))};`], ] ); diff --git a/tests/unit/integration-pool.test.js b/tests/unit/integration-pool.test.js index 9540aa1..cd26a5c 100644 --- a/tests/unit/integration-pool.test.js +++ b/tests/unit/integration-pool.test.js @@ -11,6 +11,12 @@ import { makeSlotEnv, persistDurationFile, } from "../../scripts/_integration-pool.js"; +import { + LOCAL_ADMIN_TOKEN, + LOCAL_CONNECT_HOST, + localAssetsCdnBase, + localControlUrl, +} from "../../scripts/integration-environment.js"; import { installStreamWriteCapture } from "../helpers/output-capture.js"; test("integration shard env ignores host control credentials", () => { @@ -24,10 +30,10 @@ test("integration shard env ignores host control credentials", () => { assert.equal(env.COMPOSE_PROJECT_NAME, "wdl-it-2"); assert.equal(env.WDL_GATEWAY_HOST_PORT, "18082"); - assert.equal(env.ADMIN_TOKEN, "local-dev-token"); - assert.equal(env.CONTROL_URL, "http://admin.test:18082"); - assert.equal(env.CONTROL_CONNECT_HOST, "localhost"); - assert.equal(env.ASSETS_CDN_BASE, "http://localhost:29502/wdl-assets"); + assert.equal(env.ADMIN_TOKEN, LOCAL_ADMIN_TOKEN); + assert.equal(env.CONTROL_URL, localControlUrl(18082)); + assert.equal(env.CONTROL_CONNECT_HOST, LOCAL_CONNECT_HOST); + assert.equal(env.ASSETS_CDN_BASE, localAssetsCdnBase(29502)); assert.equal(env.WDL_WORKERD_CONFIG_VARIANT, "local"); assert.equal("WDL_NS" in env, false); assert.equal("CLOUDFLARE_ENV" in env, false); diff --git a/tests/unit/integration-test-plan.test.js b/tests/unit/integration-test-plan.test.js index acb6118..c33b8ef 100644 --- a/tests/unit/integration-test-plan.test.js +++ b/tests/unit/integration-test-plan.test.js @@ -1,12 +1,17 @@ import { test } from "node:test"; import assert from "node:assert/strict"; +import { writeFileSync } from "node:fs"; import { CLI_INTEGRATION_MARKER, durationPriorityNames, hasCliIntegrationMarker, prioritizeDefaultFiles, + readIntegrationDurationRecords, + readIntegrationDurationReport, } from "../../scripts/integration-test-plan.js"; +import { installStreamWriteCapture } from "../helpers/output-capture.js"; +import { withTempDir } from "../helpers/temp-dir.js"; test("prioritizeDefaultFiles runs configured slow files first and preserves the rest", () => { assert.deepEqual( @@ -94,3 +99,40 @@ test("hasCliIntegrationMarker only accepts line-start marker comments", () => { assert.equal(hasCliIntegrationMarker(`const marker = "// ${CLI_INTEGRATION_MARKER}";\n`), false); assert.equal(hasCliIntegrationMarker(`/* ${CLI_INTEGRATION_MARKER} */\n`), false); }); + +test("integration duration reader owns the full tolerant report schema", async () => { + await withTempDir("wdl-duration-reader-", async (dir) => { + const file = `${dir}/durations.json`; + const report = { + updatedAt: "2026-07-10T00:00:00.000Z", + runDurationMs: 123, + files: { + "tests/integration/a.test.js": { + durationMs: 42, + status: "passed", + updatedAt: "2026-07-10T00:00:00.000Z", + }, + }, + }; + writeFileSync(file, JSON.stringify(report)); + + assert.deepEqual(readIntegrationDurationReport(file), report); + assert.deepEqual(readIntegrationDurationRecords(file), report.files); + }); +}); + +test("integration duration reader warns once and returns null for unreadable JSON", async () => { + await withTempDir("wdl-duration-reader-", async (dir) => { + const file = `${dir}/durations.json`; + writeFileSync(file, "not-json"); + /** @type {string[]} */ + const writes = []; + const restoreWrite = installStreamWriteCapture(process.stderr, writes); + try { + assert.equal(readIntegrationDurationReport(file), null); + } finally { + restoreWrite(); + } + assert.match(writes.join(""), /warning: ignoring unreadable integration duration file/); + }); +}); diff --git a/tests/unit/internal-auth.test.js b/tests/unit/internal-auth.test.js index 29b9712..2aad6fa 100644 --- a/tests/unit/internal-auth.test.js +++ b/tests/unit/internal-auth.test.js @@ -4,6 +4,8 @@ import { test } from "node:test"; import { INTERNAL_AUTH_HEADER, INTERNAL_AUTH_ENV, + INTERNAL_AUTH_FAILURE_CODE, + INTERNAL_AUTH_FAILURE_MESSAGE, INTERNAL_AUTH_PREVIOUS_ENV, internalAuthFailureResponse, internalAuthPreviousToken, @@ -14,10 +16,46 @@ import { withInternalAuthEntries, } from "../../shared/internal-auth.js"; import { withMockedProperty } from "../helpers/mock-global.js"; +import { readRepositoryJson } from "../helpers/load-shared-module.js"; import { assertJsonResponse } from "../helpers/response-json.js"; const TOKEN = "test-internal-auth-token"; const ENV = { [INTERNAL_AUTH_ENV]: TOKEN }; +const contract = /** @type {{ header: string, currentEnv: string, previousEnv: string, failure: { status: number, error: string, message: string }, tokenCases: Array<{ value: string, accepted: boolean }>, rotationCases: Array<{ actual: string | null, current: string, previous: string | null, accepted: boolean }>, headerCases: Array<{ values: string[], current: string, previous: string | null, accepted: boolean }> }} */ ( + readRepositoryJson("tests/fixtures/internal-auth-contract.json") +); + +test("internal auth literals and rotation match the shared Rust/JS contract", () => { + assert.equal(INTERNAL_AUTH_HEADER, contract.header); + assert.equal(INTERNAL_AUTH_ENV, contract.currentEnv); + assert.equal(INTERNAL_AUTH_PREVIOUS_ENV, contract.previousEnv); + assert.equal(INTERNAL_AUTH_FAILURE_CODE, contract.failure.error); + assert.equal(INTERNAL_AUTH_FAILURE_MESSAGE, contract.failure.message); + + for (const entry of contract.tokenCases) { + const read = () => internalAuthToken({ [INTERNAL_AUTH_ENV]: entry.value }); + if (entry.accepted) assert.equal(read(), entry.value); + else assert.throws(read); + } + for (const entry of contract.rotationCases) { + const headers = new Headers(); + if (entry.actual !== null) headers.set(INTERNAL_AUTH_HEADER, entry.actual); + const env = { + [INTERNAL_AUTH_ENV]: entry.current, + ...(entry.previous === null ? {} : { [INTERNAL_AUTH_PREVIOUS_ENV]: entry.previous }), + }; + assert.equal(verifyInternalAuthHeaders(headers, env), entry.accepted); + } + for (const entry of contract.headerCases) { + const headers = new Headers(); + for (const value of entry.values) headers.append(INTERNAL_AUTH_HEADER, value); + const env = { + [INTERNAL_AUTH_ENV]: entry.current, + ...(entry.previous === null ? {} : { [INTERNAL_AUTH_PREVIOUS_ENV]: entry.previous }), + }; + assert.equal(verifyInternalAuthHeaders(headers, env), entry.accepted); + } +}); test("internal auth token requires a non-empty configured string", () => { assert.equal(internalAuthToken(ENV), TOKEN); @@ -28,11 +66,15 @@ test("internal auth token requires a non-empty configured string", () => { assert.throws(() => internalAuthToken({ [INTERNAL_AUTH_ENV]: "" }), /WDL_INTERNAL_AUTH_TOKEN must be configured/); assert.throws( () => internalAuthToken({ [INTERNAL_AUTH_ENV]: "tokén" }), - /WDL_INTERNAL_AUTH_TOKEN must be configured as a non-empty ASCII string/ + /WDL_INTERNAL_AUTH_TOKEN must be configured as visible ASCII/ ); assert.throws( () => internalAuthPreviousToken({ [INTERNAL_AUTH_PREVIOUS_ENV]: "prévious" }), - /WDL_INTERNAL_AUTH_PREVIOUS_TOKEN must be configured as a non-empty ASCII string/ + /WDL_INTERNAL_AUTH_PREVIOUS_TOKEN must be configured as visible ASCII/ + ); + assert.throws( + () => internalAuthToken({ [INTERNAL_AUTH_ENV]: "token,other" }), + /without whitespace or commas/ ); }); @@ -110,8 +152,8 @@ test("internal auth strip removes only the internal header", () => { test("internal auth failure response has stable public shape", async () => { const response = internalAuthFailureResponse(); - await assertJsonResponse(response, 401, { - error: "internal_auth_failed", - message: "Internal authentication failed", + await assertJsonResponse(response, contract.failure.status, { + error: contract.failure.error, + message: contract.failure.message, }); }); diff --git a/tests/unit/observability.test.js b/tests/unit/observability.test.js index f4d9b0d..c586f4e 100644 --- a/tests/unit/observability.test.js +++ b/tests/unit/observability.test.js @@ -15,10 +15,39 @@ import { } from "../../shared/observability.js"; import { parseStdoutJson } from "../helpers/json-payload.js"; import { readRepositoryJson } from "../helpers/load-shared-module.js"; +import { OBSERVABILITY_NOOP_URL } from "../helpers/mocks/observability.js"; import { withCapturedConsole } from "../helpers/output-capture.js"; +import { withMockedPropertyDescriptor } from "../helpers/mock-global.js"; +import { + requestIdFromOptions, + sanitizeRequestId as sanitizeRuntimeRequestId, +} from "../../runtime/_wdl-request-id.js"; const REQUEST_ID_FIXTURES = readRepositoryJson("tests/fixtures/request-id-sanitizer.json"); const OBSERVABILITY_CONTRACT = readRepositoryJson("tests/fixtures/observability-contract.json"); +const OBSERVABILITY_NOOP = await import(OBSERVABILITY_NOOP_URL); + +test("loaded-runtime request-id sanitizer matches the shared contract", () => { + for (const fixture of REQUEST_ID_FIXTURES) { + assert.equal(sanitizeRuntimeRequestId(fixture.raw), fixture.sanitized); + } +}); + +test("loaded-runtime request-id options ignore inherited values", async () => { + await withMockedPropertyDescriptor(/** @type {any} */ (Object.prototype), "requestId", { + configurable: true, + value: "inherited-value", + }, async () => { + await withMockedPropertyDescriptor(/** @type {any} */ (Object.prototype), "requestIdProvider", { + configurable: true, + value: () => "inherited-provider", + }, async () => { + assert.equal(requestIdFromOptions({}), null); + assert.equal(requestIdFromOptions({ requestId: "own-value" }), "own-value"); + assert.equal(requestIdFromOptions({ requestIdProvider: () => "own-provider" }), "own-provider"); + }); + }); +}); test("generateRequestId produces 16 lowercase hex chars", () => { for (let i = 0; i < 20; i++) { @@ -82,6 +111,12 @@ test("sanitizeRequestId follows the cross-language fixture contract", () => { } }); +test("observability noop uses production request-id and error normalization", () => { + assert.equal(OBSERVABILITY_NOOP.sanitizeRequestId, sanitizeRequestId); + assert.equal(OBSERVABILITY_NOOP.formatError, formatError); + assert.equal(OBSERVABILITY_NOOP.ensureRequestId({ "x-request-id": "bad id" }), "rid"); +}); + test("formatError normalizes null, Error objects, and primitive throwables", () => { assert.deepEqual(formatError(null), { error_message: "Unknown error" }); assert.deepEqual(formatError(new TypeError("boom")), { @@ -89,6 +124,7 @@ test("formatError normalizes null, Error objects, and primitive throwables", () error_message: "boom", }); assert.deepEqual(formatError(42), { error_message: "42" }); + assert.deepEqual(formatError(Object.create(null)), { error_message: "Unknown error" }); }); test("formatError promotes stable code/reason fields to error_code", () => { @@ -105,6 +141,33 @@ test("formatError promotes stable code/reason fields to error_code", () => { assert.equal(formatError(reasoned).error_code, "auth_unavailable"); }); +test("formatError contains throwing Error field getters", () => { + const broken = new Error("original"); + for (const field of ["name", "message", "code"]) { + Object.defineProperty(broken, field, { + configurable: true, + get() { throw new Error(`formatter read ${field}`); }, + }); + } + Object.defineProperty(broken, "reason", { value: "tenant_failure" }); + + assert.deepEqual(formatError(broken), { + error_name: "Error", + error_message: "Unknown error", + error_code: "tenant_failure", + }); +}); + +test("formatError contains a throwing Error classification trap", () => { + const throwable = new Proxy(Object.create(null), { + getPrototypeOf() { throw new Error("prototype trap"); }, + }); + + assert.deepEqual(formatError(throwable), { + error_message: "Unknown error", + }); +}); + test("MetricsRegistry: renderPrometheus emits counter TYPE + totals", () => { const reg = new MetricsRegistry(); reg.increment("requests", { service: "test", status: "200" }); @@ -245,6 +308,53 @@ test("MetricsRegistry: cardinality warning fires per-metric-name as structured J }); }); +test("structured log envelope follows the cross-language contract", () => { + const { orderedKeys, timestampShape } = OBSERVABILITY_CONTRACT.logEnvelope; + /** @type {Array<{ name: string, priority: number, stream: string }>} */ + const levels = OBSERVABILITY_CONTRACT.logEnvelope.levels; + setLogLevel("debug"); + try { + const log = createLogger("test-svc"); + withCapturedConsole(({ stdout, stderr }) => { + for (const { name } of levels) log(name, `event_${name}`, { probe: true }); + + const emitted = [ + ...stdout.map((line) => ({ line, stream: "stdout" })), + ...stderr.map((line) => ({ line, stream: "stderr" })), + ].map((record) => ({ + ...record, + payload: parseStdoutJson(record.line, `${record.stream} log payload`), + })); + assert.equal(emitted.length, levels.length); + for (const expected of levels) { + const record = emitted.find(({ payload }) => payload.level === expected.name); + assert.ok(record, `missing ${expected.name} log line`); + assert.equal(record.stream, expected.stream, `${expected.name} stream`); + const { payload } = record; + assert.deepEqual(Object.keys(payload).slice(0, orderedKeys.length), orderedKeys); + assert.equal(payload.ts.replace(/\d/g, "0"), timestampShape); + } + }); + + for (const minimum of levels) { + setLogLevel(minimum.name); + withCapturedConsole(({ stdout, stderr }) => { + for (const { name } of levels) log(name, `gated_${name}`); + const actual = [...stdout, ...stderr] + .map((line) => parseStdoutJson(line, `minimum=${minimum.name} log payload`).level) + .toSorted(); + const expected = levels + .filter(({ priority }) => priority >= minimum.priority) + .map(({ name }) => name) + .toSorted(); + assert.deepEqual(actual, expected, `minimum=${minimum.name}`); + }); + } + } finally { + setLogLevel("info"); + } +}); + test("setLogLevel=warn suppresses info but passes error through", () => { setLogLevel("warn"); try { diff --git a/tests/unit/owner-lease.test.js b/tests/unit/owner-lease.test.js index c14c34f..a472c34 100644 --- a/tests/unit/owner-lease.test.js +++ b/tests/unit/owner-lease.test.js @@ -31,7 +31,9 @@ test("owner lease helpers parse Redis owner records", () => { assert.equal(parseOwnerRecord(null), null); assert.equal(parseOwnerRecord("{not-json"), null); assert.equal(parseOwnerRecord(JSON.stringify({ taskId: "missing-generation" })), null); + assert.equal(parseOwnerRecord(JSON.stringify({ taskId: "zero", generation: 0 })), null); assert.equal(parseOwnerRecord(JSON.stringify({ taskId: "fractional", generation: 1.5 })), null); + assert.equal(parseOwnerRecord(JSON.stringify({ taskId: "unsafe", generation: Number.MAX_SAFE_INTEGER + 1 })), null); assert.equal(parseOwnerRecord(JSON.stringify({ taskId: "bad-lease", generation: 1, leaseExpiresAt: "soon" })), null); }); @@ -68,6 +70,8 @@ test("owner lease helpers derive monotonic owner generations from Redis counters values.set("fractional", "3.5"); values.set("bad", "not-a-number"); values.set("negative", "-1"); + values.set("near-max", String(Number.MAX_SAFE_INTEGER - 1)); + values.set("max", String(Number.MAX_SAFE_INTEGER)); values.set("huge", "9007199254740992"); const session = { /** @param {string} key */ @@ -97,6 +101,16 @@ test("owner lease helpers derive monotonic owner generations from Redis counters assert.equal(await nextOwnerGeneration(session, "normal", 3), 8); assert.equal(await nextOwnerGeneration(session, "normal", 10), 11); assert.equal(await nextOwnerGeneration(session, "missing", 0), 1); + assert.equal(await nextOwnerGeneration(session, "near-max", 0), Number.MAX_SAFE_INTEGER); + assert.equal(await currentOwnerGenerationCounter(session, "max"), Number.MAX_SAFE_INTEGER); + await assert.rejects( + () => nextOwnerGeneration(session, "max", 0), + /Owner generation counter is exhausted: max/ + ); + await assert.rejects( + () => nextOwnerGeneration(session, "missing", Number.MAX_SAFE_INTEGER), + /Owner generation counter is exhausted: missing/ + ); await assert.rejects( () => nextOwnerGeneration(session, "bad", 0), /Owner generation counter is corrupt: bad/ diff --git a/tests/unit/owner-protocol.test.js b/tests/unit/owner-protocol.test.js index c831fa2..80c305f 100644 --- a/tests/unit/owner-protocol.test.js +++ b/tests/unit/owner-protocol.test.js @@ -9,6 +9,7 @@ const { ownerProtocolKeys, readOwnerRecord, readOwnerRecordWithRedisTime, + readOwnerSnapshotWithRedisTime, stageOwnerClaim, stageOwnerRelease, stageOwnerRenew, @@ -68,6 +69,25 @@ test("owner protocol helpers reject invalid Redis time from combined owner reads ); }); +test("owner protocol helpers read related owner state in the timed snapshot", async () => { + const result = await readOwnerSnapshotWithRedisTime({ + /** @param {string[]} keys */ + async getManyWithTime(keys) { + assert.deepEqual(keys, ["owner-key", "delete-lock"]); + return { + values: ["raw-owner", "whole:token"], + nowMs: 1_700_000_000_456, + }; + }, + }, "owner-key", ["delete-lock"], (/** @type {string | Uint8Array | ArrayBuffer | null | undefined} */ raw) => ({ raw })); + + assert.deepEqual(result, { + owner: { raw: "raw-owner" }, + relatedValues: ["whole:token"], + nowMs: 1_700_000_000_456, + }); +}); + test("owner protocol helpers stage claim, renew, and release writes", () => { /** @type {Array} */ const commands = []; diff --git a/tests/unit/queue-keys.test.js b/tests/unit/queue-keys.test.js new file mode 100644 index 0000000..23ed34a --- /dev/null +++ b/tests/unit/queue-keys.test.js @@ -0,0 +1,32 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { + parseConsumerKey, + parseDelayedKey, + parseStreamKey, + queueConsumerKey, + queueDelayedKey, + queueStreamKey, +} from "../../shared/queue-keys.js"; +import { readRepositoryJson } from "../helpers/load-shared-module.js"; + +const QUEUE_KEY_PARSE_CASES = readRepositoryJson("tests/fixtures/queue-key-parse.json"); + +test("queue key builders compose canonical Redis shapes", () => { + assert.equal(queueStreamKey("demo", "jobs"), "queue:demo:jobs:s"); + assert.equal(queueDelayedKey("demo", "jobs"), "queue-delayed:demo:jobs"); + assert.equal(queueConsumerKey("demo", "jobs"), "queue-consumer:demo:jobs"); +}); + +test("queue key parsers match the cross-language fixture", () => { + const parsers = { + stream: parseStreamKey, + delayed: parseDelayedKey, + consumer: parseConsumerKey, + }; + for (const [kind, parser] of Object.entries(parsers)) { + for (const { key, parsed } of QUEUE_KEY_PARSE_CASES[kind]) { + assert.deepEqual(parser(key), parsed, `${kind}:${JSON.stringify(key)}`); + } + } +}); diff --git a/tests/unit/r2-utils.test.js b/tests/unit/r2-utils.test.js index ca4e553..076ecc7 100644 --- a/tests/unit/r2-utils.test.js +++ b/tests/unit/r2-utils.test.js @@ -1,6 +1,8 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { + R2_BUCKET_NAME_RE as RUNTIME_R2_BUCKET_NAME_RE, + R2_HTTP_METADATA_FIELDS, R2_OBJECT_MAX_BUFFER_BYTES, assertR2BufferSize, encodeS3KeyPath, @@ -8,9 +10,12 @@ import { r2PhysicalKey, r2PhysicalPrefix, r2RangeAndSizeFromHeaders, + r2CacheExpiryFromHeaders, + setR2CacheExpiryHeader, stripR2PhysicalPrefix, validateR2BucketName, } from "../../runtime/r2-utils.js"; +import { R2_BUCKET_NAME_RE as CONTROL_R2_BUCKET_NAME_RE } from "../../shared/ns-pattern.js"; test("r2PhysicalPrefix scopes virtual buckets under namespace", () => { assert.equal( @@ -49,6 +54,44 @@ test("validateR2BucketName enforces prefix-safe virtual bucket names", () => { assert.throws(() => validateR2BucketName("bad/name"), /bucket_name must match/); }); +test("injected and control R2 bucket grammars stay identical", () => { + assert.equal(RUNTIME_R2_BUCKET_NAME_RE.source, CONTROL_R2_BUCKET_NAME_RE.source); + assert.equal(RUNTIME_R2_BUCKET_NAME_RE.flags, CONTROL_R2_BUCKET_NAME_RE.flags); +}); + +test("R2 HTTP metadata fields and cache expiry use one mapping", () => { + assert.deepEqual(R2_HTTP_METADATA_FIELDS, [ + ["contentType", "content-type"], + ["contentLanguage", "content-language"], + ["contentDisposition", "content-disposition"], + ["contentEncoding", "content-encoding"], + ["cacheControl", "cache-control"], + ]); + const headers = new Headers(); + setR2CacheExpiryHeader(headers, 0); + assert.equal(headers.get("expires"), "Thu, 01 Jan 1970 00:00:00 GMT"); + assert.equal(r2CacheExpiryFromHeaders(headers), 0); + assert.equal(r2CacheExpiryFromHeaders(headers, { canonical: true }), 0); + setR2CacheExpiryHeader(headers, "2026-07-12T00:00:00.000Z"); + assert.equal(headers.get("expires"), "Sun, 12 Jul 2026 00:00:00 GMT"); + headers.set("expires", "0"); + assert.equal(Number.isFinite(r2CacheExpiryFromHeaders(headers)), true); + assert.equal(r2CacheExpiryFromHeaders(headers, { canonical: true }), undefined); + headers.set("expires", "not-a-date"); + assert.equal(r2CacheExpiryFromHeaders(headers), undefined); +}); + +test("setR2CacheExpiryHeader rejects invalid expiry values", () => { + for (const value of ["", "not-a-date", new Date(NaN), true]) { + const headers = new Headers(); + assert.throws( + () => setR2CacheExpiryHeader(headers, value), + /cacheExpiry must be a valid Date or timestamp/ + ); + assert.equal(headers.has("expires"), false); + } +}); + test("encodeS3KeyPath percent-encodes key segments while preserving slashes", () => { assert.equal( encodeS3KeyPath("r2/demo/uploads/a b/?.txt"), diff --git a/tests/unit/redis-lock.test.js b/tests/unit/redis-lock.test.js new file mode 100644 index 0000000..b6abdde --- /dev/null +++ b/tests/unit/redis-lock.test.js @@ -0,0 +1,64 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { + importRepositoryModule, + importSpecifierReplacements, + repositoryFileUrl, +} from "../helpers/load-shared-module.js"; + +const { + acquireTokenLock, + createTokenLock, + releaseTokenLock, + renewTokenLock, +} = await importRepositoryModule("shared/redis-lock.js", importSpecifierReplacements({ + "shared-random-id": repositoryFileUrl("shared/random-id.js"), +})); + +test("token lock owner acquires and renews with the canonical Redis options", async () => { + /** @type {unknown[][]} */ + const calls = []; + const client = { + /** @param {unknown[]} args */ + async set(...args) { + calls.push(["set", ...args]); + return "OK"; + }, + async delIfEq() { return 1; }, + }; + const lock = createTokenLock("lock:demo"); + + assert.match(lock.token, /^[0-9a-f]{32}$/); + assert.equal(await acquireTokenLock(client, lock, { ttlSeconds: 30 }), true); + assert.equal(await renewTokenLock(client, lock, 60), true); + assert.deepEqual(calls, [ + ["set", "lock:demo", lock.token, { nx: true, ttl: 30 }], + ["set", "lock:demo", lock.token, { ttl: 60, ifeq: lock.token }], + ]); +}); + +test("token lock release is token-scoped and never masks the primary outcome", async () => { + /** @type {unknown[][]} */ + const seen = []; + const releaseError = new Error("redis unavailable"); + const client = { + async set() { return "OK"; }, + /** @param {string} key @param {string} token */ + async delIfEq(key, token) { + seen.push([key, token]); + throw releaseError; + }, + }; + + await assert.doesNotReject(() => releaseTokenLock( + client, + { key: "lock:demo", token: "token" }, + { + onError(/** @type {unknown} */ err) { + assert.equal(err, releaseError); + throw new Error("logger unavailable"); + }, + } + )); + assert.deepEqual(seen, [["lock:demo", "token"]]); +}); diff --git a/tests/unit/redis-session.test.js b/tests/unit/redis-session.test.js index 5435694..b08d3a0 100644 --- a/tests/unit/redis-session.test.js +++ b/tests/unit/redis-session.test.js @@ -331,6 +331,42 @@ test("RedisSession.getWithTime batches GET and TIME on its held socket", async ( ); }); +test("RedisSession.getManyWithTime batches related GETs and TIME on its held socket", async () => { + const socket = makeFakeSocket([ + bytes("$5\r\nowner\r\n$11\r\nwhole:token\r\n*2\r\n$10\r\n1700000000\r\n$6\r\n654321\r\n"), + ]); + const { connect, state } = scriptedConnect(socket); + const client = new RedisClient("x", { connect }); + + const result = await client.session((/** @type {any} */ s) => ( + s.getManyWithTime(["owner-key", "delete-lock"]) + )); + + assert.equal(state.count, 1); + assert.deepEqual(result, { + values: ["owner", "whole:token"], + nowMs: 1_700_000_000_654, + }); + assert.equal( + decode(socket._writes[0]), + "*2\r\n$3\r\nGET\r\n$9\r\nowner-key\r\n" + + "*2\r\n$3\r\nGET\r\n$11\r\ndelete-lock\r\n" + + "*1\r\n$4\r\nTIME\r\n" + ); +}); + +test("RedisSession.getManyWithTime rejects an empty key set before opening IO", async () => { + const session = new RedisSession("x", { + connect() { + throw new Error("unexpected connection"); + }, + }); + await assert.rejects( + session.getManyWithTime([]), + /getManyWithTime requires at least one key/ + ); +}); + test("RedisSession.hSet supports object form", async () => { const socket = makeFakeSocket([bytes(":2\r\n")]); const { connect } = scriptedConnect(socket); @@ -343,6 +379,17 @@ test("RedisSession.hSet supports object form", async () => { ); }); +test("RedisSession.set supports atomic lock options on the held socket", async () => { + const socket = makeFakeSocket([bytes("+OK\r\n")]); + const { connect } = scriptedConnect(socket); + const session = new RedisSession("x", { connect }); + await session.open(); + + assert.equal(await session.set("lock", "token", { nx: true, ttl: 30 }), "OK"); + assert.match(decode(socket._writes[0]), /SET\r\n.*lock\r\n.*token\r\n.*EX\r\n.*30\r\n.*NX\r\n/s); + await session.close(); +}); + test("RedisSession.sAdd/sRem accept scalar or array", async () => { const socket = makeFakeSocket([bytes(":1\r\n:2\r\n")]); const { connect } = scriptedConnect(socket); diff --git a/tests/unit/respond.test.js b/tests/unit/respond.test.js index 17d99c3..042c731 100644 --- a/tests/unit/respond.test.js +++ b/tests/unit/respond.test.js @@ -9,6 +9,7 @@ import { jsonInitResponse, jsonResponse, prometheusResponse, + rebuildResponseWithHeaders, sanitizeJsonErrorDetails, } from "../../shared/respond.js"; import { readRepositoryJson } from "../helpers/load-shared-module.js"; @@ -16,6 +17,22 @@ import { assertJsonResponse, readJsonResponse } from "../helpers/response-json.j const OBSERVABILITY_CONTRACT = readRepositoryJson("tests/fixtures/observability-contract.json"); +/** @param {Promise} promise */ +async function settlesWithinTestWindow(promise) { + /** @type {ReturnType | undefined} */ + let timeout; + try { + return await Promise.race([ + promise.then(() => true), + new Promise((resolve) => { + timeout = setTimeout(() => resolve(false), 100); + }), + ]); + } finally { + if (timeout !== undefined) clearTimeout(timeout); + } +} + test("discardResponseBody cancels response bodies", async () => { let cancelled = false; const response = new Response(new ReadableStream({ @@ -39,6 +56,21 @@ test("discardResponseBody ignores cancellation failures", async () => { await discardResponseBody(response); }); +test("discardResponseBody does not wait for cancellation cleanup", async () => { + let cancelled = false; + const response = new Response(new ReadableStream({ + cancel() { + cancelled = true; + return new Promise(() => {}); + }, + })); + + const settled = await settlesWithinTestWindow(discardResponseBody(response)); + + assert.equal(settled, true); + assert.equal(cancelled, true); +}); + // Node's Response doesn't accept `webSocket` in init, so the 101 branch // is covered end-to-end by tests/integration/gateway-websocket.test.js. test("echoResponseWithRequestId preserves status/body/headers and stamps x-request-id", async () => { @@ -63,6 +95,21 @@ test("echoResponseWithRequestId overwrites a pre-existing x-request-id", () => { assert.equal(out.headers.get("x-request-id"), "ours-id"); }); +test("rebuildResponseWithHeaders preserves response fields while replacing headers", async () => { + const src = new Response("hello", { + status: 202, + statusText: "Accepted", + headers: { "x-old": "hidden" }, + }); + const out = rebuildResponseWithHeaders(src, { "x-new": "visible" }); + + assert.equal(out.status, 202); + assert.equal(out.statusText, "Accepted"); + assert.equal(out.headers.get("x-old"), null); + assert.equal(out.headers.get("x-new"), "visible"); + assert.equal(await out.text(), "hello"); +}); + test("jsonResponse returns JSON with the canonical content-type", async () => { const out = jsonResponse(201, { ok: true }); assert.equal(out.headers.get("content-type"), "application/json"); diff --git a/tests/unit/runtime-binding-surface.test.js b/tests/unit/runtime-binding-surface.test.js index 2dee555..1f79981 100644 --- a/tests/unit/runtime-binding-surface.test.js +++ b/tests/unit/runtime-binding-surface.test.js @@ -6,6 +6,8 @@ import { RUNTIME_METRICS_NOOP_URL } from "../helpers/mocks/runtime-metrics.js"; import { runtimeProxyBindingStubUrl } from "../helpers/runtime-proxy-stub.js"; const PROXY_BINDING_URL = runtimeProxyBindingStubUrl(); +const SHARED_BASE64_URL = repositoryFileUrl("shared/base64.js"); +const SHARED_BOUNDED_BODY_URL = repositoryFileUrl("shared/bounded-body.js"); const SHARED_RESPOND_URL = repositoryFileUrl("shared/respond.js"); const toBytesStub = `const toBytes = (value) => { @@ -16,13 +18,7 @@ const toBytesStub = `const toBytes = (value) => { return new Uint8Array(value.buffer, value.byteOffset, value.byteLength); } throw new Error("KV put: value must be string | ArrayBuffer | typed array | ReadableStream"); -}; -const bytesToBase64 = (bytes) => { - let binary = ""; - for (const byte of bytes) binary += String.fromCharCode(byte); - return btoa(binary); -}; -const base64ToBytes = (value) => Uint8Array.from(atob(value), (ch) => ch.charCodeAt(0));`; +};`; const buildAssetUrlStub = `const buildAssetUrl = (cdnBase, prefix, path) => { if (!cdnBase) throw new Error("ASSETS.url: cdnBase is not configured"); @@ -45,8 +41,10 @@ const buildAssetUrlStub = `const buildAssetUrl = (cdnBase, prefix, path) => { test("KV host RPC surface exposes only public namespace methods", async () => { const { KV } = await importRepositoryModule("runtime/bindings/kv.js", [ [/from "cloudflare:workers";/, `from ${JSON.stringify(CLOUDFLARE_WORKERS_URL)};`], - [/import \{ base64ToBytes, bytesToBase64, toBytes \} from "runtime-lib";/, toBytesStub], + [/import \{ toBytes \} from "runtime-lib";/, toBytesStub], [/from "runtime-metrics";/, `from ${JSON.stringify(RUNTIME_METRICS_NOOP_URL)};`], + [/from "shared-base64";/, `from ${JSON.stringify(SHARED_BASE64_URL)};`], + [/from "shared-bounded-body";/, `from ${JSON.stringify(SHARED_BOUNDED_BODY_URL)};`], [/from "runtime-bindings-proxy";/, `from ${JSON.stringify(PROXY_BINDING_URL)};`], [/from "shared-respond";/, `from ${JSON.stringify(SHARED_RESPOND_URL)};`], ]); diff --git a/tests/unit/runtime-bindings-do.test.js b/tests/unit/runtime-bindings-do.test.js index 0524403..9615c71 100644 --- a/tests/unit/runtime-bindings-do.test.js +++ b/tests/unit/runtime-bindings-do.test.js @@ -5,10 +5,12 @@ import { decodeDoEnvelopeMetadata as decodeDoEnvelope } from "../helpers/do-enve import { doOwnerHintHeaders, doOwnerHintResponse, + doOwnershipErrorHeaders, tenantBodyDoOwnerHintResponse, } from "../helpers/do-owner-hint.js"; import { CLOUDFLARE_WORKERS_URL } from "../helpers/mocks/cloudflare-workers.js"; import { makeRecordingFetch, withRecordingFetch } from "../helpers/mock-fetch.js"; +import { withMockedProperty } from "../helpers/mock-global.js"; import { readJsonResponse } from "../helpers/response-json.js"; import { sharedInternalAuthUrl } from "../helpers/runtime-proxy-stub.js"; @@ -40,7 +42,7 @@ function bindingWithBackend(backend) { }); } -test("DO connect headers strip tenant protocol hop and owner hint headers", () => { +test("DO connect headers strip tenant routing headers and preserve the scope request id", () => { const request = new Request("https://tenant.workers.example/ws", { headers: { "x-wdl-do-hop-count": "99", @@ -60,18 +62,19 @@ test("DO connect headers strip tenant protocol hop and owner hint headers", () = assert.equal(headers.get("x-wdl-do-hop-count"), null); assert.equal(headers.get("x-wdl-do-owner-key"), null); assert.equal(headers.get("x-wdl-do-owner-generation"), null); - assert.equal(headers.get("x-request-id"), "tenant-rid"); + assert.equal(headers.get("x-request-id"), "scope-rid"); assert.equal(headers.get("x-wdl-do-ns"), "tenant"); assert.equal(headers.get("x-wdl-do-object-name"), "room-a"); }); -test("DO fetch requestSpec strips tenant internal owner routing headers", async () => { +test("DO fetch requestSpec strips tenant routing headers and preserves the scope request id", async () => { const { spec } = await requestSpec(new Request("https://tenant.workers.example/send", { method: "POST", headers: { "x-wdl-do-hop-count": "99", "x-wdl-do-accept-owner-hint": "1", "x-wdl-do-owner-hint": "1", + "x-wdl-do-ownership-error": "owner_fence_missing", "x-wdl-do-owner-key": "tenant", "x-wdl-do-owner-generation": "123", "x-request-id": "tenant-rid", @@ -82,19 +85,58 @@ test("DO fetch requestSpec strips tenant internal owner routing headers", async assert.equal(headers.get("x-wdl-do-hop-count"), null); assert.equal(headers.get("x-wdl-do-accept-owner-hint"), null); assert.equal(headers.get("x-wdl-do-owner-hint"), null); + assert.equal(headers.get("x-wdl-do-ownership-error"), null); assert.equal(headers.get("x-wdl-do-owner-key"), null); assert.equal(headers.get("x-wdl-do-owner-generation"), null); - assert.equal(headers.get("x-request-id"), "tenant-rid"); + assert.equal(headers.get("x-request-id"), "scope-rid"); }); -test("DO owner hint parser requires integer owner generation", () => { - assert.deepEqual(ownerHintFromHeaders(new Headers(doOwnerHintHeaders({ generation: 0 }))), { +test("DO fetch requestSpec uses captured header mutation intrinsics", async () => { + const request = new Request("https://tenant.workers.example/send", { + method: "POST", + headers: { + "x-wdl-do-owner-key": "tenant", + "x-wdl-do-ownership-error": "owner_fence_missing", + }, + body: "hello", + }); + + const { spec } = await withMockedProperty( + Headers.prototype, + "delete", + function mockedDelete() {}, + () => withMockedProperty( + Headers.prototype, + "has", + function mockedHas() { return true; }, + () => withMockedProperty( + Headers.prototype, + "set", + function mockedSet() {}, + () => requestSpec(request, "scope-rid") + ) + ) + ); + + const headers = new Headers(spec.headers); + assert.equal(headers.get("x-wdl-do-owner-key"), null); + assert.equal(headers.get("x-wdl-do-ownership-error"), null); + assert.equal(headers.get("x-request-id"), "scope-rid"); +}); + +test("DO owner hint parser requires positive safe-integer owner generation", () => { + assert.deepEqual(ownerHintFromHeaders(new Headers(doOwnerHintHeaders({ generation: 3 }))), { ownerKey: "do_0123456789abcdef0123456789abcdef:Room:shard0", taskId: "do-runtime-a", endpoint: "do-runtime-a:8788", - generation: 0, + generation: 3, }); + assert.equal(ownerHintFromHeaders(new Headers(doOwnerHintHeaders({ generation: 0 }))), null); assert.equal(ownerHintFromHeaders(new Headers(doOwnerHintHeaders({ generation: 1.5 }))), null); + assert.equal( + ownerHintFromHeaders(new Headers(doOwnerHintHeaders({ generation: 9007199254740992 }))), + null + ); const missingGeneration = new Headers(doOwnerHintHeaders({})); missingGeneration.delete("x-wdl-do-owner-generation"); assert.equal(ownerHintFromHeaders(missingGeneration), null); @@ -232,7 +274,10 @@ test("DO-to-DO fetch retries owner generation races without hint opt-in", async const binding = bindingWithBackend({ fetch: makeRecordingFetch(calls, { response: () => calls.length === 1 - ? Response.json({ error: "stale_owner_generation", message: "owner moved" }, { status: 503 }) + ? Response.json({ error: "stale_owner_generation", message: "owner moved" }, { + status: 503, + headers: doOwnershipErrorHeaders("stale_owner_generation"), + }) : new Response("retried"), }), }); @@ -262,7 +307,10 @@ test("DO-to-DO fetch ignores owner hints attached to race responses", async () = response: () => calls.length === 1 ? Response.json({ error: "stale_owner_generation", message: "owner moved" }, { status: 503, - headers: doOwnerHintResponse().headers, + headers: doOwnershipErrorHeaders( + "stale_owner_generation", + doOwnerHintResponse().headers + ), }) : new Response("retried"), }), @@ -286,7 +334,10 @@ test("DO-to-DO RPC retries owner claim races without hint opt-in", async () => { const binding = bindingWithBackend({ fetch: makeRecordingFetch(calls, { response: () => calls.length === 1 - ? Response.json({ error: "owner_claim_raced", message: "retry" }, { status: 503 }) + ? Response.json({ error: "owner_claim_raced", message: "retry" }, { + status: 503, + headers: doOwnershipErrorHeaders("owner_claim_raced"), + }) : Response.json({ ok: true, result: "retried-rpc" }), }), }); @@ -384,6 +435,7 @@ test("DO-to-DO websocket strips owner hint headers from successful upgrades", as "x-wdl-do-owner-endpoint": "do-runtime-a:8788", "x-wdl-do-owner-generation": "3", "x-wdl-do-owner-hint": "1", + "x-wdl-do-ownership-error": "owner_fence_missing", }, }); }, @@ -400,4 +452,5 @@ test("DO-to-DO websocket strips owner hint headers from successful upgrades", as assert.equal(await response.text(), "upgrade"); assert.equal(response.headers.get("x-wdl-do-owner-key"), null); assert.equal(response.headers.get("x-wdl-do-owner-hint"), null); + assert.equal(response.headers.get("x-wdl-do-ownership-error"), null); }); diff --git a/tests/unit/runtime-d1-stub.test.js b/tests/unit/runtime-d1-stub.test.js index 55db260..d81a138 100644 --- a/tests/unit/runtime-d1-stub.test.js +++ b/tests/unit/runtime-d1-stub.test.js @@ -12,8 +12,10 @@ import { } from "../helpers/load-shared-module.js"; import { runtimeProxyBindingStubUrl, sharedInternalAuthUrl } from "../helpers/runtime-proxy-stub.js"; -const OWNER_ENDPOINT_URL = repositoryFileUrl("runtime/_wdl-owner-endpoint.js"); +const OWNER_ENDPOINT_URL = repositoryFileUrl("shared/owner-endpoint.js"); const STALE_OWNER_HINT_ERRORS = [ + "owner-record-invalid", + "owner-endpoint-invalid", "task-draining", "owner-lease-expired", "owner-lease-too-short", @@ -94,7 +96,7 @@ const stubSourceReplacements = [ ], [/import \{ createOwnerHintCache \} from "runtime-owner-hint-cache";/g, ownerHintCacheSource], [ - /import \{ validOwnerEndpointForService \} from "runtime-owner-endpoint";/g, + /import \{ validOwnerEndpointForService \} from "shared-owner-endpoint";/g, `import { validOwnerEndpointForService } from ${JSON.stringify(OWNER_ENDPOINT_URL)};`, ], [/from "runtime-bindings-proxy";/g, `from ${JSON.stringify(PROXY_BINDING_URL)};`], @@ -304,80 +306,31 @@ test("D1 runtime stub: learned owner hints survive new binding instances", async }); }); -test("D1 runtime stub: owner hints preserve zero and reject fractional generations", async () => { - const { db, makeDb, calls } = makeRuntimeDb(() => d1Response({ - success: true, - results: [{ via: "router" }], - }, { - headers: { - "x-wdl-d1-owner-task-id": "task-a", - "x-wdl-d1-owner-endpoint": "d1-runtime-a:8787", - "x-wdl-d1-owner-generation": "0", - }, - })); - await db.query("all", [{ sql: "select 1", params: [] }]); - - /** @type {string[]} */ - const directCalls = []; - await withMockedFetch(async (/** @type {any} */ url) => { - directCalls.push(String(url)); - return d1Response({ success: true, results: [{ via: "owner" }] }); - }, async () => { - await makeDb().query("all", [{ sql: "select 2", params: [] }]); - }); - assert.equal(calls.length, 1); - assert.deepEqual(directCalls, ["http://d1-runtime-a:8787/internal/d1/query"]); - - clearD1OwnerHintsForTest(); - const bad = makeRuntimeDb(() => d1Response({ - success: true, - results: [], - }, { - headers: { - "x-wdl-d1-owner-task-id": "task-b", - "x-wdl-d1-owner-endpoint": "d1-runtime-b:8787", - "x-wdl-d1-owner-generation": "1.5", - }, - })); - await bad.db.query("all", [{ sql: "select 3", params: [] }]); - let unexpectedFractionalFetchCalled = false; - await withMockedFetch(async () => { - unexpectedFractionalFetchCalled = true; - return d1Response({ success: true, results: [] }); - }, async () => { - await bad.makeDb().query("all", [{ sql: "select 4", params: [] }]); - }); - assert.equal( - unexpectedFractionalFetchCalled, - false, - "expected no direct fetch for fractional generation, but fetch was called" - ); - assert.equal(bad.calls.length, 2); +test("D1 runtime stub rejects non-canonical owner generations", async () => { + for (const rawGeneration of ["0", "1.5", "9007199254740992", null]) { + clearD1OwnerHintsForTest(); + const fixture = makeRuntimeDb(() => d1Response({ + success: true, + results: [{ via: "router" }], + }, { + headers: { + "x-wdl-d1-owner-task-id": "task-a", + "x-wdl-d1-owner-endpoint": "d1-runtime-a:8787", + ...(rawGeneration == null ? {} : { "x-wdl-d1-owner-generation": rawGeneration }), + }, + })); + await fixture.db.query("all", [{ sql: "select 1", params: [] }]); - clearD1OwnerHintsForTest(); - const missing = makeRuntimeDb(() => d1Response({ - success: true, - results: [], - }, { - headers: { - "x-wdl-d1-owner-task-id": "task-c", - "x-wdl-d1-owner-endpoint": "d1-runtime-c:8787", - }, - })); - await missing.db.query("all", [{ sql: "select 5", params: [] }]); - let unexpectedMissingGenerationFetchCalled = false; - await withMockedFetch(async () => { - unexpectedMissingGenerationFetchCalled = true; - return d1Response({ success: true, results: [] }); - }, async () => { - await missing.makeDb().query("all", [{ sql: "select 6", params: [] }]); - }); - assert.equal( - unexpectedMissingGenerationFetchCalled, - false, - "expected no direct fetch without owner generation, but fetch was called" - ); - assert.equal(missing.calls.length, 2); + let unexpectedDirectFetch = false; + await withMockedFetch(async () => { + unexpectedDirectFetch = true; + return d1Response({ success: true, results: [{ via: "owner" }] }); + }, async () => { + await fixture.makeDb().query("all", [{ sql: "select 2", params: [] }]); + }); + assert.equal(unexpectedDirectFetch, false, String(rawGeneration)); + assert.equal(fixture.calls.length, 2, String(rawGeneration)); + } }); test("D1 runtime stub: accepts Kubernetes headless owner endpoints", async () => { diff --git a/tests/unit/runtime-dispatch-handlers.test.js b/tests/unit/runtime-dispatch-handlers.test.js index 6377ed7..59a4c77 100644 --- a/tests/unit/runtime-dispatch-handlers.test.js +++ b/tests/unit/runtime-dispatch-handlers.test.js @@ -258,11 +258,11 @@ test("handleFetchDispatch truncates tail path fields before append", async () => } }); -test("handleFetchDispatch emits error finish tail event while returning runtime_error", async () => { +test("handleFetchDispatch preserves falsey thrown values in the error finish tail event", async () => { const fetchSpy = installTailFetchSpy(["demo:fetch-error-worker"]); const ctx = makeCtx(); const scope = makeScope(); - const err = new Error("fetch failed"); + const err = 0; try { const res = await handleFetchDispatch({ @@ -292,7 +292,7 @@ test("handleFetchDispatch emits error finish tail event while returning runtime_ assert.equal(events[1].phase, "finish"); assert.equal(events[1].path, "/app"); assert.equal(events[1].outcome, "error"); - assert.equal(events[1].error, "fetch failed"); + assert.equal(events[1].error, "0"); } finally { fetchSpy.restore(); } @@ -379,6 +379,35 @@ test("handleQueuedDispatch decodes queue messages before JSRPC dispatch", async assert.equal(calls[0].messages[0].attempts, 3); }); +test("handleQueuedDispatch maps invalid base64 to a permanent decode failure", async () => { + let dispatchCalls = 0; + const scope = makeScope(); + const res = await handleQueuedDispatch({ + request: jsonRequest({ + queue: "jobs", + messages: [{ + id: "m1", + first_seen_ms: "123", + attempts: "0", + body_b64: "%%%", + content_type: "bytes", + }], + }), + scope, + stub: makeStub({ + async queue() { + dispatchCalls += 1; + return { ackAll: true }; + }, + }), + }); + + const body = await readJsonResponse(res, 400); + assert.equal(body.error, "queue_message_decode_failed"); + assert.match(body.message, /Invalid base64 input/); + assert.equal(dispatchCalls, 0); +}); + test("handleQueuedDispatch accepts legal queue batches above the small dispatch cap", async () => { /** @type {any[]} */ const calls = []; diff --git a/tests/unit/runtime-dispatch-workflows.test.js b/tests/unit/runtime-dispatch-workflows.test.js index 9038fbb..fe2d9f5 100644 --- a/tests/unit/runtime-dispatch-workflows.test.js +++ b/tests/unit/runtime-dispatch-workflows.test.js @@ -2,6 +2,10 @@ import { beforeEach, test } from "node:test"; import assert from "node:assert/strict"; import { parseJsonText } from "../helpers/json-payload.js"; import { loadRuntimeDispatch } from "../helpers/load-runtime-dispatch.js"; +import { + importRepositoryModule, + readRepositoryJson, +} from "../helpers/load-shared-module.js"; import { jsonRequest, makeScope, @@ -12,6 +16,10 @@ import { readJsonResponse } from "../helpers/response-json.js"; import { delay } from "../helpers/timing.js"; const { runtimeDispatch, runtimeDispatchWorkflowStep } = await loadRuntimeDispatch(); +const workflowJson = await importRepositoryModule("runtime/dispatch/workflow-json.js"); +const workflowLimits = /** @type {{ resultBytesMax: number, backendRequestBytesMax: number, payloadTooLargeCode: string }} */ ( + readRepositoryJson("tests/fixtures/workflow-limits.json") +); const { _resetWorkflowReplayCacheForTest, _stringifyWorkflowBackendBodyForTest, @@ -38,6 +46,22 @@ beforeEach(() => { _resetWorkflowReplayCacheForTest(); }); +test("workflow payload limits match the shared Rust/JS contract", () => { + assert.equal(workflowJson.WORKFLOW_RESULT_BYTES_MAX, workflowLimits.resultBytesMax); + assert.equal( + workflowJson.WORKFLOW_BACKEND_REQUEST_BYTES_MAX, + workflowLimits.backendRequestBytesMax + ); + assert.equal( + workflowJson.WORKFLOW_PAYLOAD_TOO_LARGE_CODE, + workflowLimits.payloadTooLargeCode + ); + assert.throws( + () => workflowJson._stringifyWorkflowJsonForTest("x", 0), + (err) => err instanceof Error && err.name === workflowLimits.payloadTooLargeCode + ); +}); + test("workflow bounded JSON serializer matches JSON.stringify for supported values", () => { const inherited = Object.create({ hidden: true }); inherited.visible = "yes"; diff --git a/tests/unit/runtime-do-client.test.js b/tests/unit/runtime-do-client.test.js index b6cbd60..723aca8 100644 --- a/tests/unit/runtime-do-client.test.js +++ b/tests/unit/runtime-do-client.test.js @@ -6,13 +6,30 @@ import { clearDoOwnerHintsForTest, setDoOwnerHintMaxEntriesForTest, } from "../../runtime/do-client.js"; -import { requestSpec } from "../../runtime/_wdl-do-transport.js"; +import { + MAX_DO_REQUEST_HEADER_BYTES, + dispatchDoInvokeWithHintCache, + fetchInvokeInit, + ownerHintFromHeaders, + requestSpec, + rpcInvokeBody, +} from "../../runtime/_wdl-do-transport.js"; import { decodeDoEnvelope } from "../helpers/do-envelope.js"; -import { doOwnerHintHeaders, doOwnerHintResponse } from "../helpers/do-owner-hint.js"; +import { loadDoProtocol } from "../helpers/load-do-protocol.js"; +import { + doOwnerHintHeaders, + doOwnerHintResponse, + doOwnershipErrorHeaders, +} from "../helpers/do-owner-hint.js"; import { makeRecordingFetch } from "../helpers/mock-fetch.js"; -import { withMockedProperty } from "../helpers/mock-global.js"; +import { + withMockedProperty, + withMockedPropertyDescriptor, +} from "../helpers/mock-global.js"; import { readJsonResponse } from "../helpers/response-json.js"; +const { normalizeDoInvokeRequest } = await loadDoProtocol(); + test.afterEach(() => { clearDoOwnerHintsForTest(); }); @@ -22,7 +39,48 @@ function headerValue(headers, name) { return new Headers(headers).get(name); } -test("DurableObjectNamespace facade forwards fetch with object name and request id", async () => { +/** + * @template T + * @param {Promise} promise + * @param {string} message + */ +async function withTestTimeout(promise, message) { + /** @type {ReturnType | undefined} */ + let timeout; + try { + return await Promise.race([ + promise, + new Promise((_, reject) => { + timeout = setTimeout(() => reject(new Error(message)), 1000); + }), + ]); + } finally { + if (timeout !== undefined) clearTimeout(timeout); + } +} + +/** @param {UnderlyingSourceCancelCallback} onCancel @param {ResponseInit} [init] */ +function cancellableResponse(onCancel, init = {}) { + return new Response(new ReadableStream({ + pull(controller) { + controller.error(new Error("discarded response body was read")); + }, + cancel: onCancel, + }, { highWaterMark: 0 }), init); +} + +/** + * @param {UnderlyingSourceCancelCallback} onCancel + * @param {Parameters[0]} [options] + */ +function cancellableDoOwnerHintResponse(onCancel, options = {}) { + return cancellableResponse(onCancel, { + status: 409, + headers: doOwnerHintHeaders(options), + }); +} + +test("DurableObjectNamespace fetch uses its private request-id provider", async () => { /** @type {any[]} */ const calls = []; const ns = new DurableObjectNamespace({ @@ -31,6 +89,7 @@ test("DurableObjectNamespace facade forwards fetch with object name and request return new Response("ok", { status: 201 }); }, }, { requestIdProvider: () => "rid-1" }); + Reflect.set(ns, "requestId", () => "tenant-rid"); const id = ns.idFromName("room-a"); const response = await ns.get(id).fetch("https://demo.workers.example/chat", { @@ -99,6 +158,27 @@ test("DurableObjectNamespace metadata host proxy handles normal fetch while webs assert.equal(backendCalls[0].url, "http://do-runtime/internal/do/connect"); }); +test("DurableObjectNamespace classifies metadata with a captured Object.hasOwn", async () => { + await withMockedProperty(Object, "hasOwn", () => { + throw new Error("tenant Object.hasOwn was called"); + }, async () => { + const namespace = new DurableObjectNamespace({ + ns: "tenant", + worker: "worker", + version: "v1", + doStorageId: "storage", + className: "Room", + hostProxy: { + async fetchObject() { + return new Response("ok"); + }, + }, + }); + const response = await namespace.get(namespace.idFromName("room")).fetch("https://example.test/"); + assert.equal(await response.text(), "ok"); + }); +}); + test("DurableObjectNamespace direct backend keeps binding/backend in private fields", async () => { /** @type {any[]} */ const calls = []; @@ -120,6 +200,7 @@ test("DurableObjectNamespace direct backend keeps binding/backend in private fie const response = await ns.get(ns.idFromName("room-a")).fetch("https://demo.workers.example/send", { method: "POST", + headers: { "x-request-id": "tenant-forged" }, body: "hello", }); @@ -136,6 +217,8 @@ test("DurableObjectNamespace direct backend keeps binding/backend in private fie assert.equal(body.doStorageId, "do_0123456789abcdef0123456789abcdef"); assert.equal(body.className, "Room"); assert.equal(body.objectName, "room-a"); + const forwardedRequest = /** @type {{ headers: HeadersInit }} */ (body.request); + assert.equal(new Headers(forwardedRequest.headers).get("x-request-id"), "rid-1"); assert.equal(Buffer.from(bodyBytes).toString("utf8"), "hello"); }); @@ -238,7 +321,7 @@ test("DurableObjectNamespace fetch rejects oversized invoke envelopes before tra ); }); -test("DurableObjectNamespace stub forwards arbitrary RPC methods", async () => { +test("DurableObjectNamespace RPC uses its private request-id provider", async () => { /** @type {any[]} */ const calls = []; const backend = { @@ -252,6 +335,7 @@ test("DurableObjectNamespace stub forwards arbitrary RPC methods", async () => { binding: "ROOM", className: "Room", }, { backend, requestIdProvider: () => "rid-rpc" }); + Reflect.set(ns, "requestId", () => "tenant-rid"); const stub = ns.get(ns.idFromName("room-a")); const result = await Reflect.get(stub, "addMessage")("hi", { role: "user" }); @@ -353,6 +437,89 @@ test("DurableObjectNamespace RPC rejects invalid and reserved methods before tra await assert.rejects(Reflect.get(stub, "alarm")(), /rpc\.method is reserved/); }); +test("DO RPC validation uses captured JSON and method intrinsics", async () => { + const props = { + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + className: "Room", + }; + + await withMockedProperty(Object, "entries", () => [], () => { + assert.throws( + () => rpcInvokeBody(props, "room-a", "save", [{ fn() {} }]), + (error) => error instanceof TypeError && error.message === "rpc.args[0].fn must be JSON data" + ); + }); + await withMockedProperty(Number, "isFinite", () => true, () => { + assert.throws( + () => rpcInvokeBody(props, "room-a", "save", [Number.NaN]), + (error) => error instanceof TypeError && error.message === "rpc.args[0] must be a finite number" + ); + }); + await withMockedProperty(RegExp.prototype, "test", () => true, () => { + assert.throws( + () => rpcInvokeBody(props, "room-a", "not-valid-method", []), + (error) => error instanceof TypeError && error.message === "rpc.method is not valid" + ); + }); +}); + +test("DO RPC method bounds match server normalization", () => { + const props = { + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + className: "Room", + }; + for (const { method, valid } of [ + { method: "save", valid: true }, + { method: "m".repeat(256), valid: true }, + { method: "m".repeat(257), valid: false }, + ]) { + const metadata = { + ...props, + objectName: "room-a", + kind: "rpc", + rpc: { method, args: [] }, + }; + if (!valid) { + assert.throws(() => rpcInvokeBody(props, "room-a", method, []), /rpc\.method is not valid/); + assert.throws(() => normalizeDoInvokeRequest(metadata), /rpc\.method is too large/); + continue; + } + const envelope = rpcInvokeBody(props, "room-a", method, []); + const invoke = normalizeDoInvokeRequest(decodeDoEnvelope(envelope).metadata); + assert.equal("rpc" in invoke ? invoke.rpc.method : null, method); + } +}); + +test("DO RPC snapshots tenant arguments once before sizing and encoding", () => { + const props = { + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + className: "Room", + }; + let reads = 0; + const argument = {}; + Object.defineProperty(argument, "payload", { + enumerable: true, + get() { + reads += 1; + return reads === 1 ? "stable" : "x".repeat(1_200_000); + }, + }); + + const envelope = rpcInvokeBody(props, "room-a", "save", [argument]); + const { metadata } = decodeDoEnvelope(envelope); + assert.equal(reads, 1); + assert.equal(/** @type {any} */ (metadata).rpc.args[0].payload, "stable"); +}); + test("DurableObjectNamespace RPC rejects oversized args before transport", async () => { const backend = { async fetch() { @@ -411,7 +578,10 @@ test("DurableObjectNamespace RPC retries stale owner generation once", async () fetch: makeRecordingFetch(calls, { response: () => { if (calls.length === 1) { - return Response.json({ error: "stale_owner_generation", message: "owner moved" }, { status: 503 }); + return Response.json({ error: "stale_owner_generation", message: "owner moved" }, { + status: 503, + headers: doOwnershipErrorHeaders("stale_owner_generation"), + }); } return Response.json({ ok: true, result: "rpc-ok" }); }, @@ -438,10 +608,17 @@ test("DurableObjectNamespace RPC retries stale owner generation once", async () test("DurableObjectNamespace RPC retries owner claim races once", async () => { /** @type {any[]} */ const calls = []; + let cancellations = 0; + let hostileCancelCalls = 0; const backend = { fetch: makeRecordingFetch(calls, { response: () => calls.length === 1 - ? Response.json({ error: "owner_claim_raced", message: "retry" }, { status: 503 }) + ? cancellableResponse(() => { + cancellations += 1; + }, { + status: 503, + headers: doOwnershipErrorHeaders("owner_claim_raced"), + }) : Response.json({ ok: true, result: "claimed" }), }), }; @@ -453,12 +630,24 @@ test("DurableObjectNamespace RPC retries owner claim races once", async () => { binding: "ROOM", className: "Room", }, { backend }); - - const result = await Reflect.get(ns.get(ns.idFromName("room-a")), "addMessage")("hello"); + const streamCancel = ReadableStream.prototype.cancel; + + const result = await withMockedProperty( + ReadableStream.prototype, + "cancel", + /** @this {ReadableStream} */ + function hostileCancel(reason) { + hostileCancelCalls += 1; + return Reflect.apply(streamCancel, this, [reason]); + }, + () => Reflect.get(ns.get(ns.idFromName("room-a")), "addMessage")("hello") + ); assert.equal(result, "claimed"); assert.equal(calls.length, 2); assert.equal(calls[1].init.headers.get("x-wdl-do-accept-owner-hint"), null); + assert.equal(cancellations, 1); + assert.equal(hostileCancelCalls, 0); }); test("DurableObjectNamespace direct backend preserves binary request bodies", async () => { @@ -492,7 +681,10 @@ test("DurableObjectNamespace direct backend retries stale owner generation once" fetch: makeRecordingFetch(calls, { response: () => { if (calls.length === 1) { - return Response.json({ error: "stale_owner_generation", message: "owner moved" }, { status: 503 }); + return Response.json({ error: "stale_owner_generation", message: "owner moved" }, { + status: 503, + headers: doOwnershipErrorHeaders("stale_owner_generation"), + }); } return new Response("ok"); }, @@ -507,24 +699,646 @@ test("DurableObjectNamespace direct backend retries stale owner generation once" className: "Room", }, { backend }); - const response = await ns.get(ns.idFromName("room-a")).fetch("https://demo.workers.example/send", { - method: "POST", - body: "hello", - }); + const response = await ns.get(ns.idFromName("room-a")).fetch("https://demo.workers.example/send", { + method: "POST", + body: "hello", + }); + + assert.equal(await response.text(), "ok"); + assert.equal(calls.length, 2); + assert.equal(calls[0].init.body, calls[1].init.body); + assert.equal(headerValue(calls[0].init.headers, "x-wdl-do-accept-owner-hint"), "1"); + assert.equal(calls[1].init.headers.get("x-wdl-do-accept-owner-hint"), null); +}); + +test("DurableObjectNamespace does not replay tenant responses that mimic ownership errors", async () => { + /** @type {any[]} */ + const calls = []; + const backend = { + fetch: makeRecordingFetch(calls, { + response: Response.json({ + error: "owner_fence_missing", + message: "tenant response after side effect", + }, { status: 503 }), + }), + }; + const ns = new DurableObjectNamespace({ + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + binding: "ROOM", + className: "Room", + }, { backend }); + + const originalGet = Headers.prototype.get; + await withMockedProperty(Headers.prototype, "get", /** @this {Headers} */ function get(name) { + if (String(name).toLowerCase() === "x-wdl-do-ownership-error") { + return "owner_fence_missing"; + } + return originalGet.call(this, name); + }, async () => { + const response = await ns.get(ns.idFromName("room-a")).fetch("https://demo.workers.example/send", { + method: "POST", + body: "hello", + }); + + assert.equal(response.status, 503); + assert.deepEqual(await response.json(), { + error: "owner_fence_missing", + message: "tenant response after side effect", + }); + assert.equal(calls.length, 1); + }); +}); + +test("DurableObjectNamespace owner-race classification uses the captured status getter", async () => { + /** @type {any[]} */ + const calls = []; + const backend = { + fetch: makeRecordingFetch(calls, { + response: () => calls.length === 1 + ? Response.json({ error: "owner_claim_raced", message: "retry" }, { + status: 503, + headers: doOwnershipErrorHeaders("owner_claim_raced"), + }) + : new Response("retried"), + }), + }; + const ns = new DurableObjectNamespace({ + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + binding: "ROOM", + className: "Room", + }, { backend }); + + const response = await withMockedPropertyDescriptor( + Response.prototype, + "status", + { configurable: true, get: () => 200 }, + () => ns.get(ns.idFromName("room-a")).fetch("https://demo.workers.example/send") + ); + + assert.equal(await response.text(), "retried"); + assert.equal(calls.length, 2); +}); + +test("DurableObjectNamespace retry policy uses the captured Set membership check", async () => { + /** @type {any[]} */ + const calls = []; + const backend = { + fetch: makeRecordingFetch(calls, { + response: () => calls.length === 1 + ? Response.json({ error: "owner_unavailable", message: "outcome unknown" }, { + status: 503, + headers: doOwnershipErrorHeaders("owner_unavailable"), + }) + : new Response("replayed"), + }), + }; + const ns = new DurableObjectNamespace({ + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + binding: "ROOM", + className: "Room", + }, { backend }); + + const response = await withMockedProperty( + Set.prototype, + "has", + function mockedHas() { return true; }, + () => ns.get(ns.idFromName("room-a")).fetch("https://demo.workers.example/send", { + method: "POST", + body: "side effect", + }) + ); + + assert.equal(response.status, 503); + assert.equal(calls.length, 1); +}); + +test("DurableObjectNamespace retry policy uses the captured Request method getter", async () => { + /** @type {any[]} */ + const calls = []; + const backend = { + fetch: makeRecordingFetch(calls, { + response: Response.json({ error: "owner_unavailable", message: "outcome unknown" }, { + status: 503, + headers: doOwnershipErrorHeaders("owner_unavailable"), + }), + }), + }; + const ns = new DurableObjectNamespace({ + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + binding: "ROOM", + className: "Room", + }, { backend }); + let reads = 0; + + const response = await withMockedPropertyDescriptor( + Request.prototype, + "method", + { + configurable: true, + get() { + reads += 1; + return reads === 1 ? "POST" : "GET"; + }, + }, + () => ns.get(ns.idFromName("room-a")).fetch("https://demo.workers.example/send", { + method: "POST", + body: "side effect", + }) + ); + + assert.equal(response.status, 503); + assert.equal(calls.length, 1); + assert.equal(reads, 0); + const { metadata } = decodeDoEnvelope(calls[0].init.body); + assert.equal(/** @type {any} */ (metadata.request).method, "POST"); +}); + +test("DurableObjectNamespace retry policy uses captured string case intrinsics", async () => { + /** @type {any[]} */ + const calls = []; + const backend = { + fetch: makeRecordingFetch(calls, { + response: Response.json({ error: "owner_unavailable", message: "outcome unknown" }, { + status: 503, + headers: doOwnershipErrorHeaders("owner_unavailable"), + }), + }), + }; + const ns = new DurableObjectNamespace({ + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + binding: "ROOM", + className: "Room", + }, { backend }); + let upperReads = 0; + + const response = await withMockedProperty( + String.prototype, + "toUpperCase", + function mockedToUpperCase() { + upperReads += 1; + return upperReads === 1 ? "POST" : "GET"; + }, + () => withMockedProperty( + String.prototype, + "toLowerCase", + function mockedToLowerCase() { + return "websocket"; + }, + () => ns.get(ns.idFromName("room-a")).fetch("https://demo.workers.example/send", { + method: "POST", + body: "side effect", + }) + ) + ); + + assert.equal(response.status, 503); + assert.equal(calls.length, 1); + assert.equal(upperReads, 0); + const { metadata } = decodeDoEnvelope(calls[0].init.body); + assert.equal(/** @type {any} */ (metadata.request).method, "POST"); +}); + +test("DurableObjectNamespace strips owner metadata with patched Headers methods", async () => { + const backend = { + fetch: makeRecordingFetch([], { + response: new Response("ok", { + headers: doOwnershipErrorHeaders("owner_fence_missing", doOwnerHintHeaders()), + }), + }), + }; + const ns = new DurableObjectNamespace({ + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + binding: "ROOM", + className: "Room", + }, { backend }); + + const iteratorSymbol = Symbol.iterator; + const arrayIterator = Array.prototype[Symbol.iterator]; + const arrayIncludes = Array.prototype.includes; + const response = await withMockedProperty( + Headers.prototype, + "delete", + function mockedDelete() {}, + () => withMockedProperty( + /** @type {any} */ (Array.prototype), + iteratorSymbol, + /** @this {unknown[]} */ + function targetedIterator() { + if (Reflect.apply(arrayIncludes, this, ["x-wdl-do-owner-task-id"])) { + return { + next: () => ({ done: true, value: undefined }), + [iteratorSymbol]() { return this; }, + }; + } + return Reflect.apply(arrayIterator, this, []); + }, + () => ns.get(ns.idFromName("room-a")).fetch("https://demo.workers.example/send") + ) + ); + + assert.equal(await response.text(), "ok"); + assert.equal(response.headers.get("x-wdl-do-owner-task-id"), null); + assert.equal(response.headers.get("x-wdl-do-owner-endpoint"), null); + assert.equal(response.headers.get("x-wdl-do-ownership-error"), null); +}); + +test("DO requestSpec header budget uses captured array iteration", async () => { + const iteratorSymbol = Symbol.iterator; + const arrayIterator = Array.prototype[Symbol.iterator]; + const request = new Request("https://demo.workers.example/send", { + headers: { "x-oversized": "a".repeat(MAX_DO_REQUEST_HEADER_BYTES + 1) }, + }); + + await withMockedProperty( + /** @type {any} */ (Array.prototype), + iteratorSymbol, + /** @this {unknown[]} */ + function targetedIterator() { + const first = this[0]; + if ( + (Array.isArray(first) && first[0] === "x-oversized") || + first === "x-oversized" + ) { + return { + next: () => ({ done: true, value: undefined }), + [iteratorSymbol]() { return this; }, + }; + } + return Reflect.apply(arrayIterator, this, []); + }, + async () => { + await assert.rejects( + () => requestSpec(request, null), + /fetch headers exceed 65536 bytes/ + ); + } + ); +}); + +test("DO requestSpec header budget uses captured TextEncoder.encode", async () => { + const textEncode = TextEncoder.prototype.encode; + let hostileEncodeCalls = 0; + const request = new Request("https://demo.workers.example/send", { + headers: { "x-oversized": "a".repeat(MAX_DO_REQUEST_HEADER_BYTES + 1) }, + }); + + await withMockedProperty( + TextEncoder.prototype, + "encode", + /** @this {TextEncoder} */ + function targetedEncode(value = "") { + if (value.length > MAX_DO_REQUEST_HEADER_BYTES) { + hostileEncodeCalls += 1; + return new Uint8Array(); + } + return Reflect.apply(textEncode, this, [value]); + }, + async () => { + await assert.rejects( + () => requestSpec(request, null), + /fetch headers exceed 65536 bytes/ + ); + } + ); + assert.equal(hostileEncodeCalls, 0); +}); + +function chunkedBodyRequest() { + return new Request("https://demo.workers.example/send", /** @type {RequestInit} */ (/** @type {unknown} */ ({ + method: "POST", + body: new ReadableStream({ + start(controller) { + controller.enqueue(Uint8Array.of(1, 2)); + controller.enqueue(Uint8Array.of(3, 4)); + controller.close(); + }, + }), + duplex: "half", + }))); +} + +test("DO requestSpec body collection uses captured Array.push", async () => { + const arrayPush = Array.prototype.push; + let hostilePushCalls = 0; + + const { bodyBytes } = await withMockedProperty( + Array.prototype, + "push", + /** @this {unknown[]} */ + function targetedPush(value) { + if (value instanceof Uint8Array) { + hostilePushCalls += 1; + return this.length; + } + return Reflect.apply(arrayPush, this, [value]); + }, + () => requestSpec(chunkedBodyRequest(), null) + ); + + assert.equal(hostilePushCalls, 0); + assert.deepEqual(bodyBytes, Uint8Array.of(1, 2, 3, 4)); +}); + +test("DO requestSpec body copying uses captured array iteration", async () => { + const iteratorSymbol = Symbol.iterator; + const arrayIterator = Array.prototype[Symbol.iterator]; + let hostileIteratorCalls = 0; + + const { bodyBytes } = await withMockedProperty( + /** @type {any} */ (Array.prototype), + iteratorSymbol, + /** @this {unknown[]} */ + function targetedIterator() { + if (this[0] instanceof Uint8Array) { + hostileIteratorCalls += 1; + return { + next: () => ({ done: true, value: undefined }), + [iteratorSymbol]() { return this; }, + }; + } + return Reflect.apply(arrayIterator, this, []); + }, + () => requestSpec(chunkedBodyRequest(), null) + ); + + assert.equal(hostileIteratorCalls, 0); + assert.deepEqual(bodyBytes, Uint8Array.of(1, 2, 3, 4)); +}); + +test("DO requestSpec body copying uses captured Uint8Array.set", async () => { + const uint8ArraySet = Uint8Array.prototype.set; + let hostileSetCalls = 0; + + const { bodyBytes } = await withMockedProperty( + Uint8Array.prototype, + "set", + /** @this {Uint8Array} */ + function targetedSet(source, offset) { + if (this.length === 4 && source instanceof Uint8Array) { + hostileSetCalls += 1; + return; + } + return Reflect.apply(uint8ArraySet, this, [source, offset]); + }, + () => requestSpec(chunkedBodyRequest(), null) + ); + + assert.equal(hostileSetCalls, 0); + assert.deepEqual(bodyBytes, Uint8Array.of(1, 2, 3, 4)); +}); + +test("DO requestSpec body reading uses the captured Request body getter", async () => { + let hostileBodyGetterCalls = 0; + const request = chunkedBodyRequest(); + + const { bodyBytes } = await withMockedPropertyDescriptor( + Request.prototype, + "body", + { + configurable: true, + enumerable: true, + get() { + hostileBodyGetterCalls += 1; + return null; + }, + }, + () => requestSpec(request, null) + ); + + assert.equal(hostileBodyGetterCalls, 0); + assert.deepEqual(bodyBytes, Uint8Array.of(1, 2, 3, 4)); +}); + +test("DO requestSpec body reading uses captured stream reader methods", async () => { + const getReader = ReadableStream.prototype.getReader; + const read = ReadableStreamDefaultReader.prototype.read; + const releaseLock = ReadableStreamDefaultReader.prototype.releaseLock; + let hostileMethodCalls = 0; + + const { bodyBytes } = await withMockedProperty( + ReadableStream.prototype, + "getReader", + /** @type {any} */ (/** @this {ReadableStream} @param {ReadableStreamGetReaderOptions | undefined} options */ + function targetedGetReader(options) { + hostileMethodCalls += 1; + return Reflect.apply(getReader, this, [options]); + }), + () => withMockedProperty( + ReadableStreamDefaultReader.prototype, + "read", + /** @this {ReadableStreamDefaultReader} */ + function targetedRead() { + hostileMethodCalls += 1; + return Reflect.apply(read, this, []); + }, + () => withMockedProperty( + ReadableStreamDefaultReader.prototype, + "releaseLock", + /** @this {ReadableStreamDefaultReader} */ + function targetedReleaseLock() { + hostileMethodCalls += 1; + return Reflect.apply(releaseLock, this, []); + }, + () => requestSpec(chunkedBodyRequest(), null) + ) + ) + ); + + assert.equal(hostileMethodCalls, 0); + assert.deepEqual(bodyBytes, Uint8Array.of(1, 2, 3, 4)); +}); + +test("DO invoke envelope uses captured serialization intrinsics", async () => { + const jsonStringify = JSON.stringify; + const dataViewSetUint32 = DataView.prototype.setUint32; + const uint8ArraySet = Uint8Array.prototype.set; + let hostileJsonCalls = 0; + let hostileLengthCalls = 0; + let hostileSetCalls = 0; + + const init = await withMockedProperty( + JSON, + "stringify", + /** @this {typeof JSON} */ + function targetedStringify(value) { + if (value && typeof value === "object" && "doStorageId" in value) { + hostileJsonCalls += 1; + return "{}"; + } + return Reflect.apply(jsonStringify, this, [value]); + }, + () => withMockedProperty( + DataView.prototype, + "setUint32", + /** @this {DataView} */ + function targetedSetUint32(offset, value, littleEndian) { + if (offset === 0) { + hostileLengthCalls += 1; + return; + } + return Reflect.apply(dataViewSetUint32, this, [offset, value, littleEndian]); + }, + () => withMockedProperty( + Uint8Array.prototype, + "set", + /** @this {Uint8Array} */ + function targetedSet(source, offset) { + if (offset === 4) { + hostileSetCalls += 1; + return; + } + return Reflect.apply(uint8ArraySet, this, [source, offset]); + }, + () => fetchInvokeInit({ + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + className: "Room", + }, "room-a", new Request("https://demo.workers.example/send"), null) + ) + ) + ); + + assert.equal(hostileJsonCalls, 0); + assert.equal(hostileLengthCalls, 0); + assert.equal(hostileSetCalls, 0); + const { metadata } = decodeDoEnvelope(/** @type {Uint8Array} */ (init.body)); + assert.equal(/** @type {any} */ (metadata).ns, "tenant"); + assert.equal(/** @type {any} */ (metadata).objectName, "room-a"); +}); + +test("DO invoke envelope ignores inherited object toJSON hooks", async () => { + const init = await withMockedProperty( + /** @type {any} */ (Object.prototype), + "toJSON", + function hostileToJSON() { + return { + ns: "attacker", + worker: "attacker", + version: "v9", + doStorageId: "do_ffffffffffffffffffffffffffffffff", + className: "Room", + objectName: "other", + request: { method: "GET", url: "https://evil.example/", headers: [] }, + }; + }, + () => fetchInvokeInit({ + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + className: "Room", + }, "room-a", new Request("https://demo.workers.example/send"), null) + ); + + const { metadata } = decodeDoEnvelope(/** @type {Uint8Array} */ (init.body)); + assert.equal(/** @type {any} */ (metadata).ns, "tenant"); + assert.equal(/** @type {any} */ (metadata).objectName, "room-a"); +}); + +test("DO invoke envelope ignores inherited array toJSON hooks", async () => { + const init = await withMockedProperty( + /** @type {any} */ (Array.prototype), + "toJSON", + function hostileToJSON() { + return []; + }, + () => fetchInvokeInit({ + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + className: "Room", + }, "room-a", new Request("https://demo.workers.example/send", { + headers: { "x-proof": "preserved" }, + }), null) + ); + + const { metadata } = decodeDoEnvelope(/** @type {Uint8Array} */ (init.body)); + assert.deepEqual(/** @type {any} */ (metadata).request.headers, [["x-proof", "preserved"]]); +}); - assert.equal(await response.text(), "ok"); - assert.equal(calls.length, 2); - assert.equal(calls[0].init.body, calls[1].init.body); - assert.equal(headerValue(calls[0].init.headers, "x-wdl-do-accept-owner-hint"), "1"); - assert.equal(calls[1].init.headers.get("x-wdl-do-accept-owner-hint"), null); +test("DO invoke envelope uses captured typed-array getters", async () => { + const typedArrayPrototype = /** @type {any} */ (Object.getPrototypeOf(Uint8Array.prototype)); + let hostileGetterCalls = 0; + const request = new Request("https://demo.workers.example/send"); + + const init = await withMockedPropertyDescriptor( + typedArrayPrototype, + "length", + { + configurable: true, + get() { + hostileGetterCalls += 1; + return 0; + }, + }, + () => withMockedPropertyDescriptor( + typedArrayPrototype, + "byteLength", + { + configurable: true, + get() { + hostileGetterCalls += 1; + return 0; + }, + }, + () => withMockedPropertyDescriptor( + typedArrayPrototype, + "buffer", + { + configurable: true, + get() { + hostileGetterCalls += 1; + return new ArrayBuffer(0); + }, + }, + () => fetchInvokeInit({ + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + className: "Room", + }, "room-a", request, null) + ) + ) + ); + + assert.ok(hostileGetterCalls > 0); + const { metadata } = decodeDoEnvelope(/** @type {Uint8Array} */ (init.body)); + assert.equal(/** @type {any} */ (metadata).ns, "tenant"); }); -test("DurableObjectNamespace direct backend does not retry tenant 503 responses", async () => { +test("DurableObjectNamespace direct backend retries owner claim races once", async () => { /** @type {any[]} */ const calls = []; const backend = { fetch: makeRecordingFetch(calls, { - response: Response.json({ error: "tenant_failure", message: "no retry" }, { status: 503 }), + response: () => calls.length === 1 + ? Response.json({ error: "owner_claim_raced", message: "retry" }, { + status: 503, + headers: doOwnershipErrorHeaders("owner_claim_raced"), + }) + : new Response("ok"), }), }; const ns = new DurableObjectNamespace({ @@ -536,25 +1350,33 @@ test("DurableObjectNamespace direct backend does not retry tenant 503 responses" className: "Room", }, { backend }); - const response = await ns.get(ns.idFromName("room-a")).fetch("https://demo.workers.example/send", { - method: "POST", - body: "hello", - }); + const response = await ns.get(ns.idFromName("room-a")).fetch("https://demo.workers.example/send"); - assert.equal(response.status, 503); - assert.equal(calls.length, 1); + assert.equal(await response.text(), "ok"); + assert.equal(calls.length, 2); }); -test("DurableObjectNamespace direct backend retries owner claim races once", async () => { +test("DurableObjectNamespace direct backend ignores hints attached to owner-race responses", async () => { /** @type {any[]} */ - const calls = []; + const routerCalls = []; + /** @type {any[]} */ + const ownerCalls = []; const backend = { - fetch: makeRecordingFetch(calls, { - response: () => calls.length === 1 - ? Response.json({ error: "owner_claim_raced", message: "retry" }, { status: 503 }) + fetch: makeRecordingFetch(routerCalls, { + response: () => routerCalls.length === 1 + ? Response.json({ error: "stale_owner_generation", message: "owner moved" }, { + status: 503, + headers: doOwnershipErrorHeaders( + "stale_owner_generation", + doOwnerHintResponse().headers + ), + }) : new Response("ok"), }), }; + const ownerNetwork = { + fetch: makeRecordingFetch(ownerCalls, { response: new Response("owner-should-not-be-called") }), + }; const ns = new DurableObjectNamespace({ ns: "tenant", worker: "chat", @@ -562,12 +1384,89 @@ test("DurableObjectNamespace direct backend retries owner claim races once", asy doStorageId: "do_0123456789abcdef0123456789abcdef", binding: "ROOM", className: "Room", - }, { backend }); + }, { backend, ownerNetwork }); - const response = await ns.get(ns.idFromName("room-a")).fetch("https://demo.workers.example/send"); + const response = await ns.get(ns.idFromName("room-race-hint")).fetch("https://demo.workers.example/send"); assert.equal(await response.text(), "ok"); - assert.equal(calls.length, 2); + assert.equal(routerCalls.length, 2); + assert.equal(ownerCalls.length, 0); + assert.equal(headerValue(routerCalls[0].init.headers, "x-wdl-do-accept-owner-hint"), "1"); + assert.equal(headerValue(routerCalls[1].init.headers, "x-wdl-do-accept-owner-hint"), null); +}); + +test("DO owner-race router retry failures do not trigger another replay", async (t) => { + await t.test("fresh owner response", async () => { + let routerCalls = 0; + let ownerCalls = 0; + const ownerMetadata = new Headers(doOwnerHintHeaders()); + ownerMetadata.delete("x-wdl-do-owner-hint"); + + await assert.rejects( + dispatchDoInvokeWithHintCache({ + routerFetch: async () => { + routerCalls += 1; + if (routerCalls === 1) return doOwnerHintResponse(); + if (routerCalls === 2) throw new Error("owner-race router retry failed"); + return new Response("unexpected replay"); + }, + routerUrl: "http://do-runtime/internal/do/invoke", + ownerFetch: async () => { + ownerCalls += 1; + return Response.json({ error: "owner_claim_raced", message: "retry" }, { + status: 503, + headers: doOwnershipErrorHeaders("owner_claim_raced", ownerMetadata), + }); + }, + ownerPath: "/internal/do/invoke", + init: { method: "GET" }, + cache: new Map(), + hintKey: "room-a", + replayOwnerUnavailable: true, + }), + /owner-race router retry failed/ + ); + assert.equal(ownerCalls, 1); + assert.equal(routerCalls, 2); + }); + + await t.test("cached endpoint fallback", async () => { + const hint = ownerHintFromHeaders(doOwnerHintResponse().headers); + assert.ok(hint); + const cache = new Map(); + cache.set("room-a", hint); + let routerCalls = 0; + let ownerCalls = 0; + + await assert.rejects( + dispatchDoInvokeWithHintCache({ + routerFetch: async () => { + routerCalls += 1; + if (routerCalls === 1) { + return Response.json({ error: "owner_claim_raced", message: "retry" }, { + status: 503, + headers: doOwnershipErrorHeaders("owner_claim_raced"), + }); + } + if (routerCalls === 2) throw new Error("owner-race router retry failed"); + return new Response("unexpected replay"); + }, + routerUrl: "http://do-runtime/internal/do/invoke", + ownerFetch: async () => { + ownerCalls += 1; + return new Response("owner timeout", { status: 504 }); + }, + ownerPath: "/internal/do/invoke", + init: { method: "GET" }, + cache, + hintKey: "room-a", + replayOwnerUnavailable: true, + }), + /owner-race router retry failed/ + ); + assert.equal(ownerCalls, 1); + assert.equal(routerCalls, 2); + }); }); test("DurableObjectNamespace direct backend retries owner lease budget errors once", async () => { @@ -577,7 +1476,10 @@ test("DurableObjectNamespace direct backend retries owner lease budget errors on const backend = { fetch: makeRecordingFetch(calls, { response: () => calls.length === 1 - ? Response.json({ error, message: "owner lease unavailable" }, { status: 503 }) + ? Response.json({ error, message: "owner lease unavailable" }, { + status: 503, + headers: doOwnershipErrorHeaders(error), + }) : new Response("ok"), }), }; @@ -636,6 +1538,92 @@ test("DurableObjectNamespace direct backend follows owner hints on the same requ assert.equal(ownerCalls[0].init.body, routerCalls[0].init.body); }); +test("DurableObjectNamespace replays a safe GET through the router after a second owner hint", async () => { + /** @type {any[]} */ + const routerCalls = []; + /** @type {any[]} */ + const ownerCalls = []; + /** @type {string[]} */ + const cancellations = []; + const backend = { + fetch: makeRecordingFetch(routerCalls, { + response: () => routerCalls.length === 1 + ? cancellableDoOwnerHintResponse(() => { + cancellations.push("router"); + }) + : new Response("router-ok"), + }), + }; + const ownerNetwork = { + fetch: makeRecordingFetch(ownerCalls, { + response: () => cancellableDoOwnerHintResponse( + () => { + cancellations.push("direct"); + }, + { + taskId: "do-runtime-b", + endpoint: "do-runtime-b:8788", + generation: 4, + }, + ), + }), + }; + const ns = new DurableObjectNamespace({ + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + binding: "ROOM", + className: "Room", + }, { backend, ownerNetwork }); + + const response = await ns.get(ns.idFromName("room-consecutive-hint")) + .fetch("https://demo.workers.example/send"); + assert.equal(await response.text(), "router-ok"); + assert.equal(response.headers.get("x-wdl-do-owner-key"), null); + assert.equal(routerCalls.length, 2); + assert.equal(headerValue(routerCalls[1].init.headers, "x-wdl-do-accept-owner-hint"), null); + assert.equal(ownerCalls.length, 1); + assert.deepEqual(cancellations, ["router", "direct"]); +}); + +test("DurableObjectNamespace hides a second owner hint without replaying a POST", async () => { + /** @type {any[]} */ + const routerCalls = []; + /** @type {any[]} */ + const ownerCalls = []; + const backend = { + fetch: makeRecordingFetch(routerCalls, { response: doOwnerHintResponse() }), + }; + const ownerNetwork = { + fetch: makeRecordingFetch(ownerCalls, { response: doOwnerHintResponse({ + taskId: "do-runtime-b", + endpoint: "do-runtime-b:8788", + generation: 4, + }) }), + }; + const ns = new DurableObjectNamespace({ + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + binding: "ROOM", + className: "Room", + }, { backend, ownerNetwork }); + + const response = await ns.get(ns.idFromName("room-consecutive-post")) + .fetch("https://demo.workers.example/send", { method: "POST", body: "hello" }); + const body = await readJsonResponse(response, 503); + + assert.deepEqual(body, { + error: "owner_unavailable", + message: "DO owner is unavailable; request outcome may be unknown", + }); + assert.equal(response.headers.get("x-wdl-do-owner-key"), null); + assert.equal(routerCalls.length, 1); + assert.equal(ownerCalls.length, 1); +}); + test("DurableObjectNamespace direct backend accepts Kubernetes headless owner endpoints", async () => { /** @type {any[]} */ const routerCalls = []; @@ -823,8 +1811,43 @@ test("DurableObjectNamespace direct backend accepts VPC-local IPv4 owner hint en assert.equal(ownerCalls[1].url, "http://100.64.30.52:8788/internal/do/invoke"); }); +test("DO owner hint parsing keeps the validated endpoint with patched String", async () => { + const headers = new Headers(doOwnerHintHeaders({ endpoint: "do-runtime-a:8788" })); + const nativeString = String; + let hostileStringCalls = 0; + + const hint = await withMockedProperty( + globalThis, + "String", + /** @type {StringConstructor} */ (function hostileString(value) { + if (value === "do-runtime-a:8788") { + hostileStringCalls += 1; + return "10.0.0.5:8788"; + } + return nativeString(value); + }), + () => ownerHintFromHeaders(headers) + ); + + assert.equal(hostileStringCalls, 0); + assert.equal(hint?.endpoint, "do-runtime-a:8788"); +}); + +test("DO owner hint generation parsing requires a positive safe integer with captured intrinsics", async () => { + const headers = new Headers(doOwnerHintHeaders({ endpoint: "do-runtime-a:8788" })); + await withMockedProperty(Number, "isSafeInteger", () => false, () => { + assert.equal(ownerHintFromHeaders(headers)?.generation, 3); + }); + for (const generation of ["0", "not-an-integer", "9007199254740992"]) { + headers.set("x-wdl-do-owner-generation", generation); + await withMockedProperty(Number, "isSafeInteger", () => true, () => { + assert.equal(ownerHintFromHeaders(headers), null, generation); + }); + } +}); + test("DurableObjectNamespace direct backend rejects unsafe IPv4 owner hint endpoints", async () => { - for (const endpoint of ["0.0.0.0:8788", "127.0.0.1:8788", "169.254.169.254:8788", "224.0.0.1:8788"]) { + for (const endpoint of ["0.0.0.0:8788", "8.8.8.8:8788", "127.0.0.1:8788", "169.254.169.254:8788", "224.0.0.1:8788"]) { /** @type {any[]} */ const ownerCalls = []; const backend = { @@ -914,6 +1937,146 @@ test("DurableObjectNamespace direct backend reuses learned owner hints", async ( assert.equal(ownerCalls[1].url, "http://do-runtime-a:8788/internal/do/invoke"); }); +test("DurableObjectNamespace cancels a cached-owner response before router rediscovery", async () => { + /** @type {any[]} */ + const routerCalls = []; + /** @type {any[]} */ + const ownerCalls = []; + let cancellations = 0; + const backend = { + fetch: makeRecordingFetch(routerCalls, { + response: () => routerCalls.length === 1 + ? doOwnerHintResponse() + : new Response("router-ok"), + }), + }; + const ownerNetwork = { + fetch: makeRecordingFetch(ownerCalls, { + response: () => ownerCalls.length === 1 + ? new Response("owner-ok", { headers: doOwnerHintHeaders() }) + : cancellableDoOwnerHintResponse(() => { + cancellations += 1; + }, { + taskId: "do-runtime-b", + endpoint: "do-runtime-b:8788", + generation: 4, + }), + }), + }; + const ns = new DurableObjectNamespace({ + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + binding: "ROOM", + className: "Room", + }, { backend, ownerNetwork }); + const id = ns.idFromName("room-cached-hint"); + + assert.equal(await ns.get(id).fetch("https://demo.workers.example/one").then((r) => r.text()), "owner-ok"); + assert.equal(await ns.get(id).fetch("https://demo.workers.example/two").then((r) => r.text()), "router-ok"); + + assert.equal(cancellations, 1); + assert.equal(routerCalls.length, 2); + assert.equal(ownerCalls.length, 2); +}); + +function hostileOwnerHintCacheFixture() { + /** @type {any[]} */ + const routerCalls = []; + /** @type {any[]} */ + const ownerCalls = []; + const backend = { + fetch: makeRecordingFetch(routerCalls, { response: doOwnerHintResponse() }), + }; + const ownerNetwork = { + fetch: makeRecordingFetch(ownerCalls, { response: new Response("owner-ok") }), + }; + const props = { + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + binding: "ROOM", + className: "Room", + }; + return { + ns: new DurableObjectNamespace(props, { backend, ownerNetwork }), + ownerCalls, + props, + routerCalls, + }; +} + +test("DurableObjectNamespace owner hint cache ignores tenant-patched Map#get", async () => { + const { ns, ownerCalls, props, routerCalls } = hostileOwnerHintCacheFixture(); + const objectName = "room-hostile-map-get"; + const cacheKey = `${props.doStorageId}:${props.className}:${objectName}`; + const originalMapGet = Map.prototype.get; + let hostileGetCalls = 0; + + const response = await withMockedProperty( + Map.prototype, + "get", + /** @this {Map} */ + function hostileMapGet(key) { + if (key === cacheKey) { + hostileGetCalls += 1; + return { + ownerKey: "forged-owner", + taskId: "forged-task", + endpoint: "10.0.0.5:8788", + generation: 1, + }; + } + return Reflect.apply(originalMapGet, this, [key]); + }, + () => ns.get(ns.idFromName(objectName)).fetch("https://demo.workers.example/send") + ); + + assert.equal(await response.text(), "owner-ok"); + assert.equal(hostileGetCalls, 0); + assert.equal(routerCalls.length, 1); + assert.equal(ownerCalls.length, 1); + assert.equal(ownerCalls[0].url, "http://do-runtime-a:8788/internal/do/invoke"); +}); + +test("DurableObjectNamespace owner hint cache hides hints from tenant-patched Map methods", async () => { + const { ns, ownerCalls, props, routerCalls } = hostileOwnerHintCacheFixture(); + const objectName = "room-hostile-map-set"; + const cacheKey = `${props.doStorageId}:${props.className}:${objectName}`; + const originalMapDelete = Map.prototype.delete; + const originalMapSet = Map.prototype.set; + let hostileDeleteCalls = 0; + let leakedHint = null; + + const response = await withMockedProperty( + Map.prototype, + "delete", + /** @this {Map} */ + function hostileMapDelete(key) { + if (key === cacheKey) hostileDeleteCalls += 1; + return Reflect.apply(originalMapDelete, this, [key]); + }, + () => withMockedProperty( + Map.prototype, + "set", + /** @this {Map} */ + function hostileMapSet(key, value) { + if (key === cacheKey) leakedHint = value; + return Reflect.apply(originalMapSet, this, [key, value]); + }, + () => ns.get(ns.idFromName(objectName)).fetch("https://demo.workers.example/send") + ) + ); + + assert.equal(await response.text(), "owner-ok"); + assert.equal(hostileDeleteCalls, 0); + assert.equal(leakedHint, null); + assert.equal(routerCalls.length, 1); + assert.equal(ownerCalls.length, 1); +}); + test("DurableObjectNamespace RPC reuses learned owner hints", async () => { /** @type {any[]} */ const routerCalls = []; @@ -1035,6 +2198,23 @@ test("DurableObjectNamespace direct backend rejects oversized declared request b ); }); +test("DO request body bounds use captured Number intrinsics", async () => { + const request = new Request("https://demo.workers.example/send", { + method: "POST", + headers: { "content-length": String(1024 * 1024 + 1) }, + body: "x", + }); + await withMockedProperty(globalThis, "Number", /** @type {NumberConstructor} */ (() => 0), async () => { + await assert.rejects(fetchInvokeInit({ + ns: "tenant", + worker: "chat", + version: "v1", + doStorageId: "do_0123456789abcdef0123456789abcdef", + className: "Room", + }, "room-a", request, null), /Durable Object fetch body exceeds/); + }); +}); + test("DO requestSpec rejects streaming bodies as soon as they cross the cap", async () => { let pulls = 0; const body = new ReadableStream({ @@ -1052,22 +2232,13 @@ test("DO requestSpec rejects streaming bodies as soon as they cross the cap", as body, duplex: "half", })); - /** @type {ReturnType | undefined} */ - let timeout; - - try { - await assert.rejects( - Promise.race([ - requestSpec(request, "rid-stream"), - new Promise((_, reject) => { - timeout = setTimeout(() => reject(new Error("requestSpec kept reading the oversized stream")), 1000); - }), - ]), - /Durable Object fetch body exceeds/ - ); - } finally { - if (timeout) clearTimeout(timeout); - } + await assert.rejects( + withTestTimeout( + requestSpec(request, "rid-stream"), + "requestSpec kept reading the oversized stream" + ), + /Durable Object fetch body exceeds/ + ); }); test("DO requestSpec rejects oversized streams without waiting for cancel", async () => { @@ -1084,22 +2255,24 @@ test("DO requestSpec rejects oversized streams without waiting for cancel", asyn body, duplex: "half", })); - /** @type {ReturnType | undefined} */ - let timeout; - - try { - await assert.rejects( - Promise.race([ + let hostileCatchCalls = 0; + + await withMockedProperty( + Promise.prototype, + "catch", + function hostileCatch() { + hostileCatchCalls += 1; + throw new Error("tenant Promise.catch must not run"); + }, + () => assert.rejects( + withTestTimeout( requestSpec(request, "rid-cancel"), - new Promise((_, reject) => { - timeout = setTimeout(() => reject(new Error("requestSpec waited for stream cancel")), 1000); - }), - ]), + "requestSpec waited for stream cancel" + ), /Durable Object fetch body exceeds/ - ); - } finally { - if (timeout) clearTimeout(timeout); - } + ) + ); + assert.equal(hostileCatchCalls, 0); }); test("DurableObjectNamespace facade uses direct upgrade path for websockets", async () => { @@ -1122,6 +2295,7 @@ test("DurableObjectNamespace facade uses direct upgrade path for websockets", as Connection: "Upgrade", Upgrade: "websocket", "Sec-WebSocket-Key": "abc", + "x-request-id": "tenant-forged", }, }); @@ -1273,22 +2447,28 @@ test("DurableObjectNamespace websocket path does not fall back to router after o className: "Room", }, { backend, ownerNetwork }); - const response = await ns.get(ns.idFromName("room-hint-ws-fail")).fetch("https://demo.workers.example/ws", { + let liveResponseJsonCalls = 0; + const response = await withMockedProperty(Response, "json", () => { + liveResponseJsonCalls += 1; + return new Response("forged", { status: 200 }); + }, () => ns.get(ns.idFromName("room-hint-ws-fail")).fetch("https://demo.workers.example/ws", { headers: { Connection: "Upgrade", Upgrade: "websocket", "Sec-WebSocket-Key": "abc", }, - }); + })); const body = await readJsonResponse(response, 503); assert.equal(body.error, "owner_unavailable"); assert.equal(routerCalls.length, 1); + assert.equal(liveResponseJsonCalls, 0); }); test("DurableObjectNamespace POST fetch does not replay through router after direct owner failure", async () => { /** @type {any[]} */ const routerCalls = []; + let cancellations = 0; const backend = { fetch: makeRecordingFetch(routerCalls, { response: () => routerCalls.length === 1 @@ -1298,7 +2478,10 @@ test("DurableObjectNamespace POST fetch does not replay through router after dir }; const ownerNetwork = { async fetch() { - return new Response("upstream request timeout", { status: 504 }); + return cancellableResponse(() => { + cancellations += 1; + return Promise.reject(new Error("cancel failed")); + }, { status: 504 }); }, }; const ns = new DurableObjectNamespace({ @@ -1319,6 +2502,7 @@ test("DurableObjectNamespace POST fetch does not replay through router after dir assert.equal(body.error, "owner_unavailable"); assert.equal(routerCalls.length, 1); assert.equal(headerValue(routerCalls[0].init.headers, "x-wdl-do-accept-owner-hint"), "1"); + assert.equal(cancellations, 1); }); test("DurableObjectNamespace POST drops stale cached owner hints without replay after endpoint timeout", async () => { @@ -1326,6 +2510,7 @@ test("DurableObjectNamespace POST drops stale cached owner hints without replay const routerCalls = []; /** @type {any[]} */ const ownerCalls = []; + let cancellations = 0; const backend = { fetch: makeRecordingFetch(routerCalls, { response: () => routerCalls.length === 1 @@ -1339,7 +2524,10 @@ test("DurableObjectNamespace POST drops stale cached owner hints without replay ? new Response("owner-ok", { headers: doOwnerHintHeaders(), }) - : new Response("upstream request timeout", { status: 504 }), + : cancellableResponse(() => { + cancellations += 1; + return new Promise(() => {}); + }, { status: 504 }), }), }; const ns = new DurableObjectNamespace({ @@ -1353,11 +2541,16 @@ test("DurableObjectNamespace POST drops stale cached owner hints without replay const id = ns.idFromName("room-stale-cached-owner"); assert.equal(await ns.get(id).fetch("https://demo.workers.example/one", { method: "POST", body: "first" }).then((r) => r.text()), "owner-ok"); - const body = await readJsonResponse(await ns.get(id).fetch("https://demo.workers.example/two", { method: "POST", body: "second" }), 503); + const response = await withTestTimeout( + ns.get(id).fetch("https://demo.workers.example/two", { method: "POST", body: "second" }), + "owner dispatch waited for response cancellation" + ); + const body = await readJsonResponse(response, 503); assert.equal(body.error, "owner_unavailable"); assert.equal(ownerCalls.length, 2); assert.equal(routerCalls.length, 1); + assert.equal(cancellations, 1); }); test("DurableObjectNamespace replays safe GET after stale cached owner endpoint timeout", async () => { @@ -1370,7 +2563,12 @@ test("DurableObjectNamespace replays safe GET after stale cached owner endpoint fetch: makeRecordingFetch(routerCalls, { response: () => routerCalls.length === 1 ? doOwnerHintResponse({ endpoint, generation: 11 }) - : new Response("router-ok"), + : routerCalls.length === 2 + ? Response.json({ error: "owner_claim_raced", message: "retry" }, { + status: 503, + headers: doOwnershipErrorHeaders("owner_claim_raced"), + }) + : new Response("router-ok"), }), }; const ownerNetwork = { @@ -1394,8 +2592,9 @@ test("DurableObjectNamespace replays safe GET after stale cached owner endpoint assert.equal(await ns.get(id).fetch("https://demo.workers.example/two").then((r) => r.text()), "router-ok"); assert.equal(ownerCalls.length, 2); - assert.equal(routerCalls.length, 2); + assert.equal(routerCalls.length, 3); assert.equal(headerValue(routerCalls[1].init.headers, "x-wdl-do-accept-owner-hint"), null); + assert.equal(headerValue(routerCalls[2].init.headers, "x-wdl-do-accept-owner-hint"), null); }); test("DurableObjectNamespace drops stale cached owner hints after owner lease budget errors", async () => { @@ -1404,6 +2603,7 @@ test("DurableObjectNamespace drops stale cached owner hints after owner lease bu const routerCalls = []; /** @type {any[]} */ const ownerCalls = []; + let cancellations = 0; const backend = { fetch: makeRecordingFetch(routerCalls, { response: () => routerCalls.length === 1 @@ -1415,7 +2615,12 @@ test("DurableObjectNamespace drops stale cached owner hints after owner lease bu fetch: makeRecordingFetch(ownerCalls, { response: () => ownerCalls.length === 1 ? new Response("owner-ok", { headers: doOwnerHintHeaders() }) - : Response.json({ error, message: "owner lease unavailable" }, { status: 503 }), + : cancellableResponse(() => { + cancellations += 1; + }, { + status: 503, + headers: doOwnershipErrorHeaders(error), + }), }), }; const ns = new DurableObjectNamespace({ @@ -1434,11 +2639,19 @@ test("DurableObjectNamespace drops stale cached owner hints after owner lease bu assert.equal(ownerCalls.length, 2, error); assert.equal(routerCalls.length, 2, error); assert.equal(headerValue(routerCalls[1].init.headers, "x-wdl-do-accept-owner-hint"), "1", error); + assert.equal(cancellations, 1, error); } }); -test("DurableObjectNamespace facade rejects foreign ids", () => { +test("DurableObjectNamespace facade rejects foreign ids", async () => { const ns = new DurableObjectNamespace({ fetchObject() {} }); assert.throws(() => ns.idFromName(""), /requires a non-empty string/); + for (const value of ["\ud800", "\udc00"]) { + assert.throws(() => ns.idFromName(value), /requires well-formed Unicode/); + assert.throws(() => ns.idFromString(value), /requires well-formed Unicode/); + } + await withMockedProperty(String.prototype, "isWellFormed", () => true, () => { + assert.throws(() => ns.idFromName("\ud800"), /requires well-formed Unicode/); + }); assert.throws(() => ns.get({ name: "room-a" }), /requires an id returned by this namespace/); }); diff --git a/tests/unit/runtime-do-owner-network.test.js b/tests/unit/runtime-do-owner-network.test.js index 447f7f7..e0426c7 100644 --- a/tests/unit/runtime-do-owner-network.test.js +++ b/tests/unit/runtime-do-owner-network.test.js @@ -7,15 +7,20 @@ import { repositoryFileUrl, } from "../helpers/load-shared-module.js"; import { withMockedFetch } from "../helpers/mock-fetch.js"; +import { + withMockedProperty, + withMockedPropertyDescriptor, +} from "../helpers/mock-global.js"; import { readJsonResponse } from "../helpers/response-json.js"; import { sharedInternalAuthUrl } from "../helpers/runtime-proxy-stub.js"; +import { validOwnerEndpointForService } from "../../shared/owner-endpoint.js"; -const OWNER_ENDPOINT_URL = repositoryFileUrl("runtime/_wdl-owner-endpoint.js"); +const OWNER_ENDPOINT_URL = repositoryFileUrl("shared/owner-endpoint.js"); const TEST_INTERNAL_AUTH_TOKEN = "test-internal-auth-token"; async function loadWorker() { const source = applyModuleReplacements(readRepositoryFile("runtime/do-owner-network.js"), [ - [/from "runtime-owner-endpoint";/g, `from ${JSON.stringify(OWNER_ENDPOINT_URL)};`], + [/from "shared-owner-endpoint";/g, `from ${JSON.stringify(OWNER_ENDPOINT_URL)};`], [/from "shared-internal-auth";/g, `from ${JSON.stringify(sharedInternalAuthUrl())};`], ]); return await import(moduleDataUrl(source)); @@ -59,6 +64,7 @@ test("do owner network rejects invalid owner endpoints before forwarding", async "http://d1-runtime-a:8787/internal/do/invoke", "http://do-runtime-a:8787/internal/do/invoke", "http://169.254.169.254:8788/internal/do/invoke", + "http://8.8.8.8:8788/internal/do/invoke", "http://evil.test:8788/internal/do/invoke", ]) { const response = await worker.fetch(new Request(url)); @@ -69,6 +75,77 @@ test("do owner network rejects invalid owner endpoints before forwarding", async }); }); +test("owner endpoint validation uses captured tenant-realm intrinsics", async () => { + const invalidEndpoint = "redis-proxy-user:7070"; + + let hostileUrlCalls = 0; + await withMockedProperty( + globalThis, + "URL", + /** @type {typeof URL} */ (/** @type {unknown} */ (class HostileURL { + constructor() { + hostileUrlCalls += 1; + this.host = invalidEndpoint; + this.hostname = "do-runtime-a"; + this.port = "8788"; + this.username = ""; + this.password = ""; + this.pathname = "/"; + this.search = ""; + this.hash = ""; + } + })), + () => { + assert.equal(validOwnerEndpointForService(invalidEndpoint, 8788, "do-runtime"), false); + } + ); + assert.equal(hostileUrlCalls, 0); + + let hostileGetterCalls = 0; + await withMockedPropertyDescriptor(URL.prototype, "port", { + configurable: true, + get() { + hostileGetterCalls += 1; + return "8788"; + }, + }, () => withMockedPropertyDescriptor(URL.prototype, "hostname", { + configurable: true, + get() { + hostileGetterCalls += 1; + return "do-runtime-a"; + }, + }, () => { + assert.equal(validOwnerEndpointForService(invalidEndpoint, 8788, "do-runtime"), false); + })); + assert.equal(hostileGetterCalls, 0); + + const nativeRegExpTest = RegExp.prototype.test; + let hostileRegExpCalls = 0; + await withMockedProperty(RegExp.prototype, "test", /** @this {RegExp} */ function hostileTest(value) { + if (value === "redis-proxy-user") { + hostileRegExpCalls += 1; + return true; + } + return Reflect.apply(nativeRegExpTest, this, [value]); + }, () => { + assert.equal(validOwnerEndpointForService("redis-proxy-user:8788", 8788, "do-runtime"), false); + }); + assert.equal(hostileRegExpCalls, 0); + + const nativeString = String; + let hostileStringCalls = 0; + await withMockedProperty(globalThis, "String", /** @type {StringConstructor} */ (function hostileString(value) { + if (value === 8788) { + hostileStringCalls += 1; + return "7070"; + } + return nativeString(value); + }), () => { + assert.equal(validOwnerEndpointForService("do-runtime-a:7070", 8788, "do-runtime"), false); + }); + assert.equal(hostileStringCalls, 0); +}); + test("do owner network rejects non-owner-dispatch paths before forwarding", async () => { const { default: worker } = await loadWorker(); let called = false; diff --git a/tests/unit/runtime-internal-auth.test.js b/tests/unit/runtime-internal-auth.test.js index c6de062..9653402 100644 --- a/tests/unit/runtime-internal-auth.test.js +++ b/tests/unit/runtime-internal-auth.test.js @@ -50,8 +50,8 @@ export async function readWorkflowNotifyDispatch() { throw new Error("unexpected export async function readWorkflowRunDispatch() { throw new Error("unexpected workflow run body read"); } `); const runtimeLoadUrl = moduleDataUrl(` -export function createLoaderCallback() { - return async () => { throw new Error("unexpected worker load"); }; +export function getLoadedWorkerStub() { + throw new Error("unexpected worker load"); } `); const emptyBindingUrl = moduleDataUrl(` diff --git a/tests/unit/runtime-kv-binding.test.js b/tests/unit/runtime-kv-binding.test.js index 3d1bc68..4d8b3f8 100644 --- a/tests/unit/runtime-kv-binding.test.js +++ b/tests/unit/runtime-kv-binding.test.js @@ -13,7 +13,9 @@ import { runtimeProxyBindingStubUrl } from "../helpers/runtime-proxy-stub.js"; const PROXY_BINDING_URL = runtimeProxyBindingStubUrl(); const RUNTIME_LIB_URL = runtimeLibModuleDataUrl(); +const SHARED_BASE64_URL = repositoryFileUrl("shared/base64.js"); const SHARED_RESPOND_URL = repositoryFileUrl("shared/respond.js"); +const SHARED_BOUNDED_BODY_URL = repositoryFileUrl("shared/bounded-body.js"); /** @param {Array<[RegExp | string, string]>} [replacements] */ async function loadKvBinding(replacements = []) { @@ -21,10 +23,12 @@ async function loadKvBinding(replacements = []) { [/from "cloudflare:workers";/, `from ${JSON.stringify(CLOUDFLARE_WORKERS_URL)};`], [/from "runtime-lib";/, `from ${JSON.stringify(RUNTIME_LIB_URL)};`], [/from "runtime-metrics";/, `from ${JSON.stringify(RUNTIME_METRICS_NOOP_URL)};`], + [/from "shared-base64";/, `from ${JSON.stringify(SHARED_BASE64_URL)};`], [ /from "runtime-bindings-proxy";/, `from ${JSON.stringify(PROXY_BINDING_URL)};`, ], + [/from "shared-bounded-body";/, `from ${JSON.stringify(SHARED_BOUNDED_BODY_URL)};`], [/from "shared-respond";/, `from ${JSON.stringify(SHARED_RESPOND_URL)};`], ]; return importRepositoryModule("runtime/bindings/kv.js", /** @type {Array<[RegExp | string, string]>} */ ([...baseReplacements, ...replacements])); diff --git a/tests/unit/runtime-lib.test.js b/tests/unit/runtime-lib.test.js index 76b96bb..e180753 100644 --- a/tests/unit/runtime-lib.test.js +++ b/tests/unit/runtime-lib.test.js @@ -1,9 +1,11 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { parseBase64Json } from "../helpers/json-payload.js"; -import { runtimeLibModuleDataUrl } from "../helpers/load-shared-module.js"; +import { readRepositoryJson, runtimeLibModuleDataUrl } from "../helpers/load-shared-module.js"; const { + base64ToBytes, + bytesToBase64, toBytes, bundleToWorkerCode, buildAssetUrl, @@ -16,9 +18,26 @@ const { normalizeQueuedDispatchBody, normalizeScheduledDispatchBody, } = await import(runtimeLibModuleDataUrl()); +const sharedBase64 = await import("../../shared/base64.js"); +const versionFixture = readRepositoryJson("tests/fixtures/version-tags.json"); const enc = new TextEncoder(); +test("runtime base64 exports use the shared codec owner", () => { + assert.equal(bytesToBase64, sharedBase64.bytesToBase64); + assert.equal(base64ToBytes, sharedBase64.base64ToBytes); + const bytes = Uint8Array.from({ length: 70_000 }, (_, index) => index % 256); + assert.deepEqual(base64ToBytes(bytesToBase64(bytes)), bytes); +}); + +test("shared base64 decoder matches atob rejection semantics with Buffer present", () => { + assert.deepEqual(base64ToBytes("Z g =="), new Uint8Array([0x66])); + assert.deepEqual(base64ToBytes("Zg"), new Uint8Array([0x66])); + for (const invalid of ["%%%", "Zg=", "Zm9v=", "-_8=", "YWJjZA==x"]) { + assert.throws(() => base64ToBytes(invalid), /Invalid base64 input/); + } +}); + test("toBytes: string → Uint8Array utf8", () => { const r = toBytes("hi"); assert.ok(r instanceof Uint8Array); @@ -175,6 +194,19 @@ test("normalizeWorkflowNotifyBody validates route worker identity grammar", () = ); }); +test("workflow dispatch version ingress matches the shared JS/Rust fixture", () => { + for (const { tag, parsed } of versionFixture.cases) { + const body = workflowBody({ frozenVersion: tag }); + if (parsed == null) { + assert.throws(() => normalizeWorkflowRunBody(body), /frozenVersion/, tag); + assert.throws(() => normalizeWorkflowNotifyBody(body), /frozenVersion/, tag); + } else { + assert.equal(normalizeWorkflowRunBody(body).frozenVersion, tag, tag); + assert.equal(normalizeWorkflowNotifyBody(body).frozenVersion, tag, tag); + } + } +}); + test("decodeQueuedDispatchMessages decodes runtime queue wire messages", () => { const messages = decodeQueuedDispatchMessages([ { @@ -297,13 +329,29 @@ test("bundleToWorkerCode: uses meta.compatibilityDate when set", () => { mkBundle( { mainModule: "w.js", - compatibilityDate: "2024-01-01", + compatibilityDate: "2026-04-01", modules: { "w.js": { type: "module" } }, }, { "w.js": enc.encode("x") } ) ); - assert.equal(code.compatibilityDate, "2024-01-01"); + assert.equal(code.compatibilityDate, "2026-04-01"); +}); + +test("bundleToWorkerCode: rejects compatibilityDate below the WDL dynamic-worker floor", () => { + assert.throws( + () => bundleToWorkerCode( + mkBundle( + { + mainModule: "w.js", + compatibilityDate: "2026-03-31", + modules: { "w.js": { type: "module" } }, + }, + { "w.js": enc.encode("x") } + ) + ), + /meta\.compatibilityDate 2026-03-31 is older than WDL supports \(2026-04-01\)/ + ); }); test("bundleToWorkerCode: compatibilityFlags merge user-declared with old-date platform floor, meta is frozen", () => { @@ -369,6 +417,33 @@ test("bundleToWorkerCode: compatibilityFlags already includes floor → no dup", assert.deepEqual(code.compatibilityFlags, ["enhanced_error_serialization", "nodejs_compat"]); }); +test("bundleToWorkerCode: incompatible error serialization flags fail closed", () => { + /** @type {[string | undefined, string, RegExp][]} */ + const cases = [ + ["2026-04-20", "legacy_error_serialization", /requires enhanced error serialization/], + ["2026-04-21", "legacy_error_serialization", /requires enhanced error serialization/], + ["2026-04-21", "enhanced_error_serialization", /became the workerd default/], + [undefined, "enhanced_error_serialization", /became the workerd default/], + ]; + for (const [compatibilityDate, flag, message] of cases) { + assert.throws( + () => bundleToWorkerCode( + mkBundle( + { + mainModule: "w.js", + ...(compatibilityDate ? { compatibilityDate } : {}), + compatibilityFlags: [flag], + modules: { "w.js": { type: "module" } }, + }, + { "w.js": enc.encode("x") } + ) + ), + message, + `${flag} should be rejected for ${compatibilityDate ?? "the default date"}` + ); + } +}); + test("bundleToWorkerCode: throws (doesn't silently drop) on malformed compatibilityFlags in bundle bytes", () => { assert.throws( () => bundleToWorkerCode( @@ -610,6 +685,10 @@ test("decodeQueueBody: bytes contentType preserves binary payload", () => { assert.deepEqual(Array.from(decoded), Array.from(payload)); }); +test("decodeQueueBody: invalid base64 fails closed before handler dispatch", () => { + assert.throws(() => decodeQueueBody("%%%", "bytes"), /Invalid base64 input/); +}); + test("decodeQueueBody: unknown contentType throws (v8 path must not silently pass)", () => { assert.throws( () => decodeQueueBody(b64FromUtf8("x"), "v8"), diff --git a/tests/unit/runtime-load.test.js b/tests/unit/runtime-load.test.js index fff1ccf..9ee0e13 100644 --- a/tests/unit/runtime-load.test.js +++ b/tests/unit/runtime-load.test.js @@ -6,13 +6,16 @@ import { pathToFileURL } from "node:url"; import { importRepositoryModule, importSpecifierReplacements, - moduleDataUrl, + readRepositoryJson, readRepositoryModuleSource, repositoryFileUrl, + repositoryModuleDataUrl, } from "../helpers/load-shared-module.js"; +import { compileControlGraph } from "../helpers/load-control-lib.js"; import { makeRecordingFetch, withMockedFetch } from "../helpers/mock-fetch.js"; import { withMockedProperty } from "../helpers/mock-global.js"; import { OBSERVABILITY_NOOP_URL } from "../helpers/mocks/observability.js"; +import { sharedRedisStubUrl } from "../helpers/mocks/fake-redis.js"; import { assertJsonResponse } from "../helpers/response-json.js"; import { sharedInternalAuthUrl } from "../helpers/runtime-proxy-stub.js"; import { @@ -26,20 +29,22 @@ const SHARED_WORKER_ID_URL = repositoryFileUrl("shared/worker-id.js"); const SHARED_INTERNAL_AUTH_URL = sharedInternalAuthUrl(); const SHARED_RESPOND_URL = repositoryFileUrl("shared/respond.js"); const SHARED_SECRET_ENVELOPE_URL = repositoryFileUrl("shared/secret-envelope.js"); -const SHARED_ERRORS_URL = repositoryFileUrl("shared/errors.js"); const SHARED_VERSION_URL = repositoryFileUrl("shared/version.js"); -const SHARED_REDIS_URL = moduleDataUrl(` -export class WatchError extends Error { - constructor(message = "watched key changed") { - super(message); - this.name = "WatchError"; - } -} -`); +const SHARED_REDIS_URL = sharedRedisStubUrl(); +const { libUrl: CONTROL_LIB_URL } = await compileControlGraph(); +const RUNTIME_ENV_BUILD_URL = repositoryModuleDataUrl( + "runtime/load/env-build.js", + importSpecifierReplacements({ + "shared-ns-pattern": SHARED_NS_PATTERN_URL, + "shared-version": SHARED_VERSION_URL, + }) +); +const versionFixture = readRepositoryJson("tests/fixtures/version-tags.json"); const FETCH_STUB = { fetch() {} }; const { estimatedWorkerLoaderEnv } = await importRepositoryModule("control/env-budget.js", importSpecifierReplacements({ + "control-lib": CONTROL_LIB_URL, + "runtime-load-env-build": RUNTIME_ENV_BUILD_URL, "shared-secret-envelope": SHARED_SECRET_ENVELOPE_URL, - "shared-errors": SHARED_ERRORS_URL, "shared-version": SHARED_VERSION_URL, "shared-redis": SHARED_REDIS_URL, })); @@ -68,6 +73,15 @@ const src = readRepositoryModuleSource("runtime/load.js", [ "shared-internal-auth": SHARED_INTERNAL_AUTH_URL, "shared-respond": SHARED_RESPOND_URL, }), + [ + /import \{ evictSiblings, recordLoadedWorker \} from "runtime-state";/, + `const recordLoadedWorker = (workerId) => { + /** @type {any[]} */ (globalThis).__runtimeLoadRecordedWorkers.push(workerId); + }; + const evictSiblings = async (options) => { + /** @type {any[]} */ (globalThis).__runtimeLoadEvictions.push(options); + };` + ], [ /from "runtime-load-code-budget";/, 'from "./load/code-budget.js";' @@ -95,6 +109,7 @@ const LOAD_TEST_RUNTIME_DIR = path.join(LOAD_TEST_DIR, "runtime"); const LOAD_TEST_SUBMODULE_DIR = path.join(LOAD_TEST_RUNTIME_DIR, "load"); const ENV_BUILD_SOURCE = readRepositoryModuleSource("runtime/load/env-build.js", importSpecifierReplacements({ "shared-ns-pattern": SHARED_NS_PATTERN_URL, + "shared-version": SHARED_VERSION_URL, })); mkdirSync(LOAD_TEST_SUBMODULE_DIR, { recursive: true }); writeFileSync(path.join(LOAD_TEST_RUNTIME_DIR, "load.js"), src); @@ -113,12 +128,16 @@ for (const name of ["env-build.js", "code-budget.js", "module-rewrite.js", "wrap } after(() => removeTempDir(LOAD_TEST_DIR)); +/** @type {any} */ (globalThis).__runtimeLoadRecordedWorkers = []; +/** @type {any} */ (globalThis).__runtimeLoadEvictions = []; + const mod = await import(pathToFileURL(path.join(LOAD_TEST_RUNTIME_DIR, "load.js")).href); const codeBudgetMod = await import(pathToFileURL(path.join(LOAD_TEST_SUBMODULE_DIR, "code-budget.js")).href); const { buildWorkerEnv, createLoaderCallback, decodeRuntimeLoadPayload, + getLoadedWorkerStub, runtimeLoadContentTypeMatches, wrapWorkerCodeForHostBindings, } = mod; @@ -625,6 +644,7 @@ test("buildWorkerEnv: service binding with requiredCallerSecrets filters from ns makeCtx() ); // Listed keys only, worker wins over ns, missing key silently absent. + assert.equal(Object.getPrototypeOf(env.JSJ.props.callerSecrets), Object.prototype); assert.deepEqual(env.JSJ.props.callerSecrets, { JSJ_API_KEY: "worker-level-key" }); assert.equal(env.JSJ.props.callerNs, "demo"); assert.equal(env.JSJ.props.targetNs, "__platform__"); @@ -653,6 +673,7 @@ test("buildWorkerEnv: requiredCallerSecrets does NOT fall back to vars", () => { "https://assets.example", makeCtx() ); + assert.equal(Object.getPrototypeOf(env.JSJ.props.callerSecrets), Object.prototype); assert.deepEqual(env.JSJ.props.callerSecrets, {}); }); @@ -742,7 +763,36 @@ test("buildWorkerEnv: D1 binding requires databaseId", () => { "https://assets.example", makeCtx() ), - /D1 binding but missing databaseId/ + /D1 binding but has invalid databaseId/ + ); +}); + +test("buildWorkerEnv: D1 bindings revalidate namespace and database id grammar", () => { + assert.throws( + () => buildWorkerEnv( + { bindings: { DB: { type: "d1", databaseId: "db/child" } } }, + {}, + {}, + "demo", + "app", + "v1", + "https://assets.example", + makeCtx() + ), + /invalid databaseId/ + ); + assert.throws( + () => buildWorkerEnv( + { bindings: { DB: { type: "d1", databaseId: "main" } } }, + {}, + {}, + "admin", + "app", + "v1", + "https://assets.example", + makeCtx() + ), + /invalid namespace/ ); }); @@ -780,6 +830,26 @@ test("buildWorkerEnv: service binding requires service and version", () => { ); }); +test("buildWorkerEnv: stored service binding versions match the shared fixture", () => { + for (const { tag, parsed } of versionFixture.cases) { + if (parsed != null || tag === "") continue; + assert.throws( + () => buildWorkerEnv( + { bindings: { AUTH: { type: "service", service: "auth", version: tag } } }, + {}, + {}, + "demo", + "app", + "v1", + "https://assets.example", + makeCtx() + ), + /service binding but has invalid version/, + tag + ); + } +}); + test("buildWorkerEnv: stored service binding cannot target runtime-reserved entrypoints", () => { assert.throws( () => @@ -1161,6 +1231,77 @@ test("createLoaderCallback: attaches configured tail worker and always wraps mai ); }); +test("getLoadedWorkerStub records cache misses before scheduling active-version eviction", async () => { + /** @type {Array<{ id: string, factory: () => Promise }>} */ + const loaderCalls = []; + /** @type {Promise[]} */ + const backgroundTasks = []; + /** @type {any} */ (globalThis).__runtimeLoadRecordedWorkers.length = 0; + /** @type {any} */ (globalThis).__runtimeLoadEvictions.length = 0; + const stub = { getEntrypoint() {} }; + const env = { + REDIS_PROXY_URL: "http://redis-proxy.local", + WDL_INTERNAL_AUTH_TOKEN: "test-internal-auth-token", + ASSETS_CDN_BASE: "https://assets.example", + PUBLIC_NETWORK: { kind: "public-network" }, + LOADER: { + get(/** @type {string} */ id, /** @type {() => Promise} */ factory) { + loaderCalls.push({ id, factory }); + return stub; + }, + }, + }; + const ctx = { + ...makeCtx(), + waitUntil(/** @type {Promise} */ promise) { + backgroundTasks.push(promise); + }, + }; + + await withMockedFetch( + async () => new Response(encodeRuntimeLoadPayload({ + bundle: { + "__meta__": JSON.stringify({ + mainModule: "worker.js", + modules: { "worker.js": { type: "module" } }, + }), + "worker.js": "export default {};", + }, + }), { + status: 200, + headers: { "content-type": RUNTIME_LOAD_CONTENT_TYPE }, + }), + async () => { + const loaded = getLoadedWorkerStub({ + requestId: "rid-loader-wrapper", + env, + ctx, + ns: "demo", + worker: "app", + version: "v2", + evictOnLoad: true, + }); + + assert.equal(loaded.workerId, "demo:app:v2"); + assert.equal(loaded.stub, stub); + assert.equal(loaderCalls.length, 1); + assert.equal(loaderCalls[0].id, "demo:app:v2"); + assert.deepEqual(/** @type {any} */ (globalThis).__runtimeLoadRecordedWorkers, []); + + await loaderCalls[0].factory(); + await Promise.all(backgroundTasks); + + assert.deepEqual(/** @type {any} */ (globalThis).__runtimeLoadRecordedWorkers, ["demo:app:v2"]); + assert.deepEqual( + /** @type {any} */ (globalThis).__runtimeLoadEvictions.map( + (/** @type {{ workerId: string }} */ entry) => entry.workerId + ), + ["demo:app:v2"] + ); + } + ); +}); + test("createLoaderCallback: DO bindings are host proxies and workflow auth stays out of generated facade code", async () => { /** @type {Array<{ service: string, headers: HeadersInit | undefined }>} */ const privateCalls = []; @@ -1546,21 +1687,23 @@ test("workerLoader code estimator does not rewrite CommonJS workflow strings", ( }); test("wrapWorkerCodeForHostBindings rejects empty reserved module collisions", () => { - const workerCode = { - mainModule: "worker.js", - modules: { - "worker.js": "export default {};", - "_wdl-wrapper.js": "", - }, - }; - - assert.throws( - () => injectRuntimeModulesForHostBindings(workerCode, { + for (const reserved of ["_wdl-wrapper.js", "_wdl-host-wrapper-runtime.js"]) { + const workerCode = { mainModule: "worker.js", - modules: { "worker.js": { type: "module" } }, - }, STUB_RUNTIME_INJECTION_SOURCES), - /reserved module names/ - ); + modules: { + "worker.js": "export default {};", + [reserved]: "", + }, + }; + + assert.throws( + () => injectRuntimeModulesForHostBindings(workerCode, { + mainModule: "worker.js", + modules: { "worker.js": { type: "module" } }, + }, STUB_RUNTIME_INJECTION_SOURCES), + /reserved module names/ + ); + } }); test("wrapWorkerCodeForHostBindings: injects local D1 client wrapper and preserves original main module", () => { @@ -1594,18 +1737,40 @@ test("wrapWorkerCodeForHostBindings: injects local D1 client wrapper and preserv assert.match(/** @type {any} */ (workerCode.modules)["_wdl-d1-transport.js"], /from "\.\/_wdl-d1-data-field\.js";/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-request-id.js"], /requestIdFromOptions/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-d1-client.js"], /class D1Database/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /import \* as user from "\.\/src\/worker\.js";/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /Local wrapper subclasses intentionally shadow/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-host-wrapper-runtime.js"], /intrinsicArrayForEach/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-host-wrapper-runtime.js"], /AsyncLocalStorage/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /import \* as __WdlHostRuntime__/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /import \* as __WdlUserModule__ from "\.\/src\/worker\.js";/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /Explicit aliases replace same-name star exports/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /const D1_BINDINGS = \["DB"\];/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /const R2_BINDINGS = \[\];/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /const DO_BINDINGS = \[\];/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /new D1Database\(out\[name\], requestIdOptions\(requestIdOrContext\)\)/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /new D1Database\(out\[name\], requestIdOptions\(\)\)/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /requestIdFromEventArg/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /wrapClassInstance\(this, requestContext\)/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /const HOST_WRAPPED_HANDLER_KEYS = \["fetch", "scheduled", "queue", "tail"\]/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /export class Api extends user\.Api/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /export class Admin extends user\.Admin/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /wrapClassInstance\(this\)/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /forEachArray\(HOST_WRAPPED_HANDLER_KEYS/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /class extends __WdlUserModule__\.Api/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /__WdlWrappedEntrypoint\d+__ as Admin/); assert.doesNotMatch(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /class default/); + assert.deepEqual(/** @type {any} */ (workerCode).compatibilityFlags, ["nodejs_als"]); +}); + +test("wrapWorkerCodeForHostBindings: normalizes standalone ALS compatibility flags", () => { + for (const [input, expected] of [ + [["no_nodejs_als"], ["nodejs_als"]], + [["no_global_navigator", "no_nodejs_als"], ["no_global_navigator", "nodejs_als"]], + [["nodejs_compat", "no_nodejs_als"], ["nodejs_compat", "no_nodejs_als"]], + ]) { + const workerCode = { + compatibilityFlags: input, + mainModule: "worker.js", + modules: { "worker.js": "export default {};" }, + }; + wrapWorkerCodeForHostBindings(workerCode, { + bindings: { DB: { type: "d1", databaseId: "main" } }, + }); + assert.deepEqual(workerCode.compatibilityFlags, expected); + } }); test("wrapWorkerCodeForHostBindings: default object wraps inherited and accessor handler keys", async () => { @@ -1670,18 +1835,34 @@ test("wrapWorkerCodeForHostBindings: declared named entrypoints are wrapper subc mainModule: "worker.js", modules: { "worker.js": ` + import * as hostRuntime from "./_wdl-host-wrapper-runtime.js"; + export function hostRuntimeContextExports() { + return { + currentRequestId: typeof hostRuntime.currentRequestId, + runWithRequestContext: typeof hostRuntime.runWithRequestContext, + }; + } export class Api { constructor(ctx, env) { this.env = env; } dbConstructorName() { return this.env.DB?.constructor?.name; } rawDbVisible() { return this.env.DB?.raw === true; } + echo(value) { return value; } } + export class currentRequestId extends Api {} + export class runWithRequestContext extends Api {} + export class __wdlRunWithRequestContext extends Api {} export default {}; `, }, }; wrapWorkerCodeForHostBindings(workerCode, { bindings: { DB: { type: "d1", databaseId: "main" } }, - exports: [{ entrypoint: "Api", allowedCallers: ["*"] }], + exports: [ + { entrypoint: "Api", allowedCallers: ["*"] }, + { entrypoint: "currentRequestId", allowedCallers: ["*"] }, + { entrypoint: "runWithRequestContext", allowedCallers: ["*"] }, + { entrypoint: "__wdlRunWithRequestContext", allowedCallers: ["*"] }, + ], }); await withTempDir("wdl-d1-wrapper-", async (dir) => { @@ -1696,7 +1877,8 @@ test("wrapWorkerCodeForHostBindings: declared named entrypoints are wrapper subc for (const [name, source] of Object.entries(workerCode.modules)) { const file = path.join(dir, name); const stubbed = name === "_wdl-wrapper.js" - ? source.replace(`from "cloudflare:workers"`, `from "./_cf_workers_stub.js"`) + ? source + .replace(`from "cloudflare:workers"`, `from "./_cf_workers_stub.js"`) : source; writeFileSync(file, stubbed); } @@ -1712,6 +1894,25 @@ test("wrapWorkerCodeForHostBindings: declared named entrypoints are wrapper subc }); assert.equal(instance.dbConstructorName(), "D1Database"); assert.equal(instance.rawDbVisible(), false); + for (const name of ["currentRequestId", "runWithRequestContext", "__wdlRunWithRequestContext"]) { + assert.ok(new wrapped[name]({}, { DB: { raw: true } }) instanceof user[name]); + } + + assert.deepEqual(wrapped.hostRuntimeContextExports(), { + currentRequestId: "undefined", + runWithRequestContext: "undefined", + }); + + let thenReads = 0; + const stateful = {}; + Object.defineProperty(stateful, "then", { + get() { + thenReads += 1; + return undefined; + }, + }); + assert.equal(instance.echo(stateful), stateful); + assert.equal(thenReads, 0); }); }); @@ -1737,6 +1938,7 @@ test("wrapWorkerCodeForHostBindings: workers without host facades get only the a assert.equal(/** @type {any} */ (workerCode.modules)["_wdl-d1-client.js"], undefined); assert.equal(/** @type {any} */ (workerCode.modules)["_wdl-r2-client.js"], undefined); assert.equal(/** @type {any} */ (workerCode.modules)["_wdl-do-client.js"], undefined); + assert.equal(/** @type {any} */ (workerCode.modules)["_wdl-host-wrapper-runtime.js"], undefined); assert.equal(/** @type {any} */ (workerCode.modules)["_wdl-do-transport.js"], undefined); assert.equal(/** @type {any} */ (workerCode.modules)["_wdl-request-id.js"], undefined); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-cloudflare-workflows.js"], /class NonRetryableError extends Error/); @@ -1853,7 +2055,7 @@ test("wrapWorkerCodeForHostBindings: injects local R2 facade for R2 bindings", ( assert.match(/** @type {any} */ (workerCode.modules)["_wdl-request-id.js"], /requestIdFromOptions/); assert.doesNotMatch(/** @type {any} */ (workerCode.modules)["_wdl-r2-client.js"], /this\._stub/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /const R2_BINDINGS = \["BUCKET"\];/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /new R2Bucket\(out\[name\], requestIdOptions\(requestIdOrContext\)\)/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /new R2Bucket\(out\[name\], requestIdOptions\(\)\)/); }); test("wrapWorkerCodeForHostBindings: injects local DO facade for Durable Object bindings", () => { @@ -1872,9 +2074,9 @@ test("wrapWorkerCodeForHostBindings: injects local DO facade for Durable Object assert.match(/** @type {any} */ (workerCode.modules)["_wdl-owner-endpoint.js"], /validOwnerEndpointForService/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-request-id.js"], /requestIdFromOptions/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /const DO_BINDINGS = \["ROOMS"\];/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /new DurableObjectNamespace\(out\[name\], doOptions\(requestIdOrContext, doBackend, doOwnerNetwork\)\)/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /new DurableObjectNamespace\(out\[name\], doOptions\(doBackend, doOwnerNetwork\)\)/); assert.doesNotMatch(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /internalAuthToken|__WDL_INTERNAL_AUTH_TOKEN__/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /export class Room extends user\.Room/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /class extends __WdlUserModule__\.Room/); assert.doesNotMatch(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /export \* from/); }); @@ -1908,11 +2110,11 @@ test("wrapWorkerCodeForHostBindings: injects local Workflow facade and wraps wor assert.match(/** @type {any} */ (workerCode.modules)["_wdl-cloudflare-workflows.js"], /this\.name = "NonRetryableError"/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /import \{ Workflow \}/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /const WORKFLOW_BINDINGS = \{"ORDERS":/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /new Workflow\(out\[name\] \|\| metadata, workflowOptions\(requestIdOrContext, workflowsBackend\)\)/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /new Workflow\(out\[name\] \|\| metadata, workflowOptions\(workflowsBackend\)\)/); assert.doesNotMatch(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /internalAuthToken|__WDL_INTERNAL_AUTH_TOKEN__/); assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /notifyWorkflowCallback/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /notifyWorkflowCallback\(request, wrapEnv\(this\.env, requestIdFromEventArg\(request\)\)\)/); - assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /export class OrderWorkflow extends user\.OrderWorkflow/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /withRequestContext\(request, \(\) => notifyWorkflowCallback\(request, wrapEnv\(this\.env\)\)\)/); + assert.match(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /class extends __WdlUserModule__\.OrderWorkflow/); assert.doesNotMatch(/** @type {any} */ (workerCode.modules)["_wdl-wrapper.js"], /export \* from "\.\/worker\.js";/); }); diff --git a/tests/unit/runtime-loaded-facade-surface.test.js b/tests/unit/runtime-loaded-facade-surface.test.js index a7f2509..c2dc119 100644 --- a/tests/unit/runtime-loaded-facade-surface.test.js +++ b/tests/unit/runtime-loaded-facade-surface.test.js @@ -1,6 +1,7 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { importRepositoryModule } from "../helpers/load-shared-module.js"; +import { importRepositoryModule, repositoryFileUrl } from "../helpers/load-shared-module.js"; +import { withMockedProperty } from "../helpers/mock-global.js"; const requestIdFromOptionsStub = `const requestIdFromOptions = (options) => { if (!options || typeof options !== "object") return null; @@ -20,13 +21,8 @@ const d1Facade = await importRepositoryModule("runtime/d1-client.js", [ const r2Facade = await importRepositoryModule("runtime/r2-client.js", [ [ - /import \{\s*R2_OBJECT_MAX_BUFFER_BYTES,\s*assertR2BufferSize,\s*normalizeR2ListLimit,\s*normalizeR2ObjectKey,\s*\} from "\.\/_wdl-r2-utils\.js";/, - `const R2_OBJECT_MAX_BUFFER_BYTES = 26214400; - const assertR2BufferSize = (size, operation) => { - if (size > R2_OBJECT_MAX_BUFFER_BYTES) throw new TypeError("R2 " + operation + ": object too large"); - }; - const normalizeR2ListLimit = (limit) => limit == null ? undefined : Number(limit); - const normalizeR2ObjectKey = (key) => String(key);`, + /from "\.\/_wdl-r2-utils\.js";/, + `from ${JSON.stringify(repositoryFileUrl("runtime/r2-utils.js"))};`, ], [/import \{ requestIdFromOptions \} from "\.\/_wdl-request-id\.js";/, requestIdFromOptionsStub], ]); @@ -83,3 +79,20 @@ test("loaded R2 facade does not expose the runtime RPC stub handle", () => { "resumeMultipartUpload", ]); }); + +test("loaded D1 and R2 facades use WeakMap intrinsics captured before tenant evaluation", async () => { + const fail = () => { + throw new Error("tenant WeakMap method was called"); + }; + await withMockedProperty(WeakMap.prototype, "get", fail, () => + withMockedProperty(WeakMap.prototype, "has", fail, () => + withMockedProperty(WeakMap.prototype, "set", fail, async () => { + const db = new d1Facade.D1Database({ query: async () => ({ results: [] }) }); + assert.deepEqual(await db.prepare("SELECT 1").all(), { results: [] }); + + const bucket = new r2Facade.R2Bucket({ head: async () => null }); + assert.equal(await bucket.head("key"), null); + }) + ) + ); +}); diff --git a/tests/unit/runtime-pool-boundary.test.js b/tests/unit/runtime-pool-boundary.test.js index 0075a4f..04ab3f9 100644 --- a/tests/unit/runtime-pool-boundary.test.js +++ b/tests/unit/runtime-pool-boundary.test.js @@ -54,8 +54,9 @@ export async function readWorkflowNotifyDispatch(request) { return { body: await export async function readWorkflowRunDispatch(request) { return { body: await request.json() }; } `); const runtimeLoadUrl = moduleDataUrl(` -export function createLoaderCallback() { - return async () => ({}); +export function getLoadedWorkerStub({ env, ns, worker, version }) { + const workerId = \`\${ns}:\${worker}:\${version}\`; + return { workerId, stub: env.LOADER.get(workerId, async () => ({})) }; } `); const emptyBindingUrl = moduleDataUrl(` diff --git a/tests/unit/runtime-r2-client.test.js b/tests/unit/runtime-r2-client.test.js index 0776247..fa7e07a 100644 --- a/tests/unit/runtime-r2-client.test.js +++ b/tests/unit/runtime-r2-client.test.js @@ -3,6 +3,28 @@ import assert from "node:assert/strict"; import { R2Bucket, R2Object, R2ObjectBody } from "../../runtime/r2-client.js"; import { R2_OBJECT_MAX_BUFFER_BYTES } from "../../runtime/r2-utils.js"; +/** + * @param {Promise} promise + * @returns {Promise<{ status: "fulfilled", value: unknown } | { status: "rejected", reason: unknown } | { status: "pending" }>} + */ +async function settlementWithinTestWindow(promise) { + /** @type {ReturnType | undefined} */ + let timeout; + try { + return await Promise.race([ + promise.then( + (value) => ({ status: /** @type {const} */ ("fulfilled"), value }), + (reason) => ({ status: /** @type {const} */ ("rejected"), reason }), + ), + new Promise((resolve) => { + timeout = setTimeout(() => resolve({ status: /** @type {const} */ ("pending") }), 100); + }), + ]); + } finally { + if (timeout !== undefined) clearTimeout(timeout); + } +} + test("R2Bucket.list validates limit before host binding call", async () => { const bucket = new R2Bucket({ async list() { @@ -133,6 +155,85 @@ test("R2Bucket.put preserves onlyIf etag arrays for host binding", async () => { assert.deepEqual(calls[0].requestMeta, { requestId: "rid-put" }); }); +test("R2Bucket.put normalizes every Headers httpMetadata field", async () => { + /** @type {any[]} */ + const calls = []; + const bucket = new R2Bucket({ + async put(/** @type {string} */ _key, /** @type {any} */ _value, /** @type {any} */ options) { + calls.push(options); + return null; + }, + }); + const headers = new Headers({ + "content-type": "text/plain", + "content-language": "en", + "content-disposition": "inline", + "content-encoding": "gzip", + "cache-control": "max-age=60", + expires: "Thu, 01 Jan 1970 00:00:00 GMT", + }); + + await bucket.put("a.txt", "hello", { httpMetadata: headers }); + + assert.deepEqual(calls[0].httpMetadata, { + contentType: "text/plain", + contentLanguage: "en", + contentDisposition: "inline", + contentEncoding: "gzip", + cacheControl: "max-age=60", + cacheExpiry: 0, + }); +}); + +test("R2Bucket.put rejects non-canonical Headers expiry before the host call", async () => { + let calls = 0; + const bucket = new R2Bucket({ + async put() { + calls += 1; + return null; + }, + }); + for (const expires of [ + "not-a-date", + "0", + "2026-07-13T00:00:00.000Z", + "July 13, 2026", + "Tue, 13 Jul 2026 00:00:00 GMT", + ]) { + const headers = new Headers({ expires }); + await assert.rejects( + () => bucket.put("a.txt", "hello", { httpMetadata: headers }), + /Expires header must be canonical IMF-fixdate/ + ); + } + assert.equal(calls, 0); +}); + +test("R2Object.writeHttpMetadata writes every mapped field including epoch expiry", () => { + const object = new R2Object({ + httpMetadata: { + contentType: "text/plain", + contentLanguage: "en", + contentDisposition: "inline", + contentEncoding: "gzip", + cacheControl: "max-age=60", + cacheExpiry: 0, + }, + }); + const headers = new Headers(); + + object.writeHttpMetadata(headers); + + assert.deepEqual(Object.fromEntries(headers), { + "cache-control": "max-age=60", + "content-disposition": "inline", + "content-encoding": "gzip", + "content-language": "en", + "content-type": "text/plain", + expires: "Thu, 01 Jan 1970 00:00:00 GMT", + }); +}); + test("R2Bucket.put rejects timestamp onlyIf conditions", async () => { const bucket = new R2Bucket({ async put() { @@ -219,6 +320,33 @@ test("R2Bucket.put keeps single-chunk ReadableStream bytes without re-copying", assert.equal(observed, chunk); }); +test("R2Bucket.put rejects an oversized stream without waiting for cancel", async () => { + let cancelled = false; + let hostCalls = 0; + const bucket = new R2Bucket({ + async put() { + hostCalls += 1; + return null; + }, + }); + const stream = new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(R2_OBJECT_MAX_BUFFER_BYTES + 1)); + }, + cancel() { + cancelled = true; + return new Promise(() => {}); + }, + }); + + const outcome = await settlementWithinTestWindow(bucket.put("huge.bin", stream)); + + assert.equal(outcome.status, "rejected"); + assert.match(String("reason" in outcome ? outcome.reason : ""), /exceeds the 25 MiB WDL R2 limit/); + assert.equal(cancelled, true); + assert.equal(hostCalls, 0); +}); + test("R2Bucket.get returns R2Object when host binding returns no body", async () => { const bucket = new R2Bucket({ async get() { @@ -331,6 +459,36 @@ test("R2ObjectBody raw body stream enforces the object byte cap", async () => { ); }); +test("R2ObjectBody byte cap does not wait for underlying cancel", async () => { + let cancelled = false; + const obj = new R2ObjectBody({ + key: "huge.bin", + version: "", + size: R2_OBJECT_MAX_BUFFER_BYTES + 1, + etag: "abc", + httpEtag: '"abc"', + uploaded: Date.now(), + httpMetadata: {}, + customMetadata: {}, + checksums: {}, + storageClass: "Standard", + }, new ReadableStream({ + start(controller) { + controller.enqueue(new Uint8Array(R2_OBJECT_MAX_BUFFER_BYTES + 1)); + }, + cancel() { + cancelled = true; + return new Promise(() => {}); + }, + })); + + const outcome = await settlementWithinTestWindow(obj.body.getReader().read()); + + assert.equal(outcome.status, "rejected"); + assert.match(String("reason" in outcome ? outcome.reason : ""), /exceeds the 25 MiB WDL R2 limit/); + assert.equal(cancelled, true); +}); + test("R2Bucket multipart upload methods fail with WDL-specific errors", () => { const bucket = new R2Bucket({}); diff --git a/tests/unit/runtime-r2-host.test.js b/tests/unit/runtime-r2-host.test.js index 0fec3ec..b99c1b8 100644 --- a/tests/unit/runtime-r2-host.test.js +++ b/tests/unit/runtime-r2-host.test.js @@ -121,6 +121,33 @@ test("R2 host put returns metadata from PUT response without a follow-up HEAD", } }); +test("R2 host reads every mapped HTTP metadata response header", async () => { + const restore = installR2FetchMock(async () => new Response(null, { + status: 200, + headers: { + "content-type": "text/plain", + "content-language": "en", + "content-disposition": "inline", + "content-encoding": "gzip", + "cache-control": "max-age=60", + expires: "Thu, 01 Jan 1970 00:00:00 GMT", + }, + })); + try { + const meta = await makeR2Bucket().head("metadata.txt"); + assert.deepEqual(meta.httpMetadata, { + contentType: "text/plain", + contentLanguage: "en", + contentDisposition: "inline", + contentEncoding: "gzip", + cacheControl: "max-age=60", + cacheExpiry: 0, + }); + } finally { + restore(); + } +}); + test("R2 host metadata preserves magic custom metadata keys as data fields", async () => { const restore = installR2FetchMock(async () => new Response(null, { status: 200, diff --git a/tests/unit/runtime-service-binding.test.js b/tests/unit/runtime-service-binding.test.js index 9ed634a..99004f1 100644 --- a/tests/unit/runtime-service-binding.test.js +++ b/tests/unit/runtime-service-binding.test.js @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { test } from "node:test"; -import { importRepositoryModule } from "../helpers/load-shared-module.js"; +import { importRepositoryModule, repositoryFileUrl } from "../helpers/load-shared-module.js"; import { CLOUDFLARE_WORKERS_URL } from "../helpers/mocks/cloudflare-workers.js"; import { readJsonResponse } from "../helpers/response-json.js"; @@ -14,20 +14,23 @@ const { ServiceBinding } = await importRepositoryModule("runtime/bindings/servic `const INTERNAL_AUTH_HEADER = ${JSON.stringify(TEST_INTERNAL_AUTH_HEADER)};` ], [ - /import \{ createLoaderCallback \} from "runtime-load";/, - `const createLoaderCallback = (options) => { - /** @type {any} */ (globalThis).__serviceBindingLoaderCallbackOptions.push(options); - return async () => ({}); - };` + /import \{ sanitizeRequestId \} from "shared-observability";/, + `import { sanitizeRequestId } from ${JSON.stringify(repositoryFileUrl("shared/observability.js"))};` ], [ - /import \{ recordLoadedWorker \} from "runtime-state";/, - `const recordLoadedWorker = () => {};` + /import \{ getLoadedWorkerStub \} from "runtime-load";/, + `const getLoadedWorkerStub = (options) => { + /** @type {any} */ (globalThis).__serviceBindingLoaderCallbackOptions.push(options); + return { + workerId: options.workerId, + stub: options.env.LOADER.get(options.workerId, async () => ({})), + }; + };` ], [ - /import \{ emitRuntimeTailEvent, fetchTailFields \} from "runtime-tail-forwarder";/, - `const emitRuntimeTailEvent = () => null; - const fetchTailFields = () => ({});` + /import \{ fetchTailFields, startTailEnvelope \} from "runtime-tail-forwarder";/, + `const fetchTailFields = () => ({}); + const startTailEnvelope = () => ({ finish() {}, finishError() {} });` ], ]); @@ -102,6 +105,7 @@ test("ServiceBinding RPC cold-load does not mint a service-binding request id", const options = loaderCallbackOptions(); assert.equal(options.length, 1); assert.equal(options[0].requestId, null); + assert.equal(options[0].evictOnLoad, undefined); }); test("ServiceBinding fetch strips caller-supplied platform forwarding headers", async () => { @@ -143,3 +147,19 @@ test("ServiceBinding fetch cold-load propagates request id when present", async assert.equal(options.length, 1); assert.equal(options[0].requestId, requestId); }); + +test("ServiceBinding fetch canonicalizes request ids without minting replacements", async () => { + resetLoaderCallbackOptions(); + const binding = makeServiceBinding(); + + const canonical = await readJsonResponse(await binding.fetch(new Request("https://worker.workers.example/rpc", { + headers: { "x-request-id": " rid-first , rid-second" }, + })), 200); + assert.equal(canonical.requestId, "rid-first"); + + const invalid = await readJsonResponse(await binding.fetch(new Request("https://worker.workers.example/rpc", { + headers: { "x-request-id": "bad\\id" }, + })), 200); + assert.equal(invalid.requestId, null); + assert.equal(loaderCallbackOptions().at(-1).requestId, null); +}); diff --git a/tests/unit/runtime-tail-worker.test.js b/tests/unit/runtime-tail-worker.test.js index f53f041..6a13cf5 100644 --- a/tests/unit/runtime-tail-worker.test.js +++ b/tests/unit/runtime-tail-worker.test.js @@ -4,6 +4,7 @@ import { applyModuleReplacements, moduleDataUrl, readRepositoryFile, + repositoryFileUrl, } from "../helpers/load-shared-module.js"; import { installMockFetch, makeRecordingFetch } from "../helpers/mock-fetch.js"; import { parseJsonObjectRequestBody } from "../helpers/request-body.js"; @@ -12,6 +13,7 @@ import { delay } from "../helpers/timing.js"; const PROXY_BINDING_URL = runtimeProxyBindingStubUrl(); const SHARED_INTERNAL_AUTH_URL = sharedInternalAuthUrl(); +const SHARED_ERRORS_URL = repositoryFileUrl("shared/errors.js"); const OBSERVABILITY_STUB_SRC = `const createLogger = (service) => (level, event, fields) => /** @type {any} */ (globalThis).__runtimeTailLogs.push({ service, level, event, fields }); const createLogLevelBinder = () => { let logLevelSet = false; @@ -23,6 +25,7 @@ const createLogLevelBinder = () => { };`; const forwarderSrc = applyModuleReplacements(readRepositoryFile("runtime/tail-forwarder.js"), [ [/from "runtime-bindings-proxy";/, `from ${JSON.stringify(PROXY_BINDING_URL)};`], + [/from "shared-errors";/, `from ${JSON.stringify(SHARED_ERRORS_URL)};`], [/from "shared-internal-auth";/, `from ${JSON.stringify(SHARED_INTERNAL_AUTH_URL)};`], ]); const tailWorkerRawSrc = applyModuleReplacements(readRepositoryFile("runtime/tail-worker.js"), [ diff --git a/tests/unit/runtime-workflows-client.test.js b/tests/unit/runtime-workflows-client.test.js index af4ceca..099511f 100644 --- a/tests/unit/runtime-workflows-client.test.js +++ b/tests/unit/runtime-workflows-client.test.js @@ -2,8 +2,13 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { Workflow } from "../../runtime/workflows-client.js"; +import { readRepositoryJson } from "../helpers/load-shared-module.js"; import { parseJsonObjectRequestBody } from "../helpers/request-body.js"; +const workflowLimits = /** @type {{ createBatchMax: number }} */ ( + readRepositoryJson("tests/fixtures/workflow-limits.json") +); + /** * @param {Record} [runtimeOptions] * @param {Record} [metadataOverrides] @@ -65,6 +70,7 @@ test("Workflow.create preserves runtime metadata from params override", async () assert.equal(fetchCalls, 1); assert.equal(capturedUrl, "http://workflows/internal/workflows/create"); assert.equal(capturedInit?.method, "POST"); + assert.equal(new Headers(capturedInit?.headers).get("x-request-id"), "runtime-request"); assert.equal(typeof capturedInit?.body, "string"); assert.deepEqual(capturedRequestBody, { ns: "tenant-a", @@ -229,6 +235,16 @@ test("Workflow.create supports omitted requestId in context", async () => { assert.equal(capturedBody.requestId, null); }); +test("Workflow.createBatch limit matches the cross-language fixture", async () => { + const workflow = createWorkflowForTest(); + await assert.rejects( + () => workflow.createBatch(Array.from({ length: workflowLimits.createBatchMax + 1 }, (_, index) => ({ + id: `inst-${index}`, + }))), + new RegExp(`exceeds ${workflowLimits.createBatchMax} item limit`), + ); +}); + test("Workflow.createBatch accepts backend-skipped ids", async () => { /** @type {Record | undefined} */ let capturedBody; diff --git a/tests/unit/runtime-wrapper-generate.test.js b/tests/unit/runtime-wrapper-generate.test.js new file mode 100644 index 0000000..902a697 --- /dev/null +++ b/tests/unit/runtime-wrapper-generate.test.js @@ -0,0 +1,277 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { AsyncLocalStorage } from "node:async_hooks"; +import { + HOST_BINDING_RUNTIME_MODULE_NAME, + HOST_BINDING_RUNTIME_SOURCE, + generateAbortShimWrapperModule, + generateHostBindingWrapperModule, +} from "../../runtime/load/wrapper-generate.js"; +import { + applyModuleReplacements, + moduleDataUrl, + repositoryFileUrl, +} from "../helpers/load-shared-module.js"; +import { + installMockProperty, + withMockedProperty, + withMockedPropertyDescriptor, +} from "../helpers/mock-global.js"; + +const HOST_BINDING_RUNTIME_TEST_SOURCE = applyModuleReplacements(HOST_BINDING_RUNTIME_SOURCE, [ + [ + 'from "./_wdl-request-id.js"', + `from ${JSON.stringify(repositoryFileUrl("runtime/_wdl-request-id.js"))}`, + ], +]); +const hostBindingRuntime = await import(moduleDataUrl(HOST_BINDING_RUNTIME_TEST_SOURCE)); +const hostRequestContext = hostBindingRuntime.createRequestContext(); + +function generatedWrappers() { + return { + abortOnly: generateAbortShimWrapperModule("worker.js"), + hostBindings: generateHostBindingWrapperModule("worker.js", [], [], [], {}, []), + }; +} + +/** + * @param {string} source + * @param {string} startMarker + * @param {string} endMarker + */ +function sourceFragment(source, startMarker, endMarker) { + const start = source.indexOf(startMarker); + assert.notEqual(start, -1, `missing generated source marker: ${startMarker}`); + const end = source.indexOf(endMarker, start); + assert.notEqual(end, -1, `missing generated source marker: ${endMarker}`); + return source.slice(start, end); +} + +test("generated wrapper flavors share the exact abort shim", () => { + const { abortOnly, hostBindings } = generatedWrappers(); + const start = "// Reserved name (WDL_RESERVED_ENTRYPOINT_RE)"; + const end = "\n\nexport class __WdlWorkflowNotify__"; + + assert.equal( + sourceFragment(abortOnly, start, end), + sourceFragment(hostBindings, start, end) + ); +}); + +test("generated wrapper flavors preserve default-export class detection", () => { + const { abortOnly, hostBindings } = generatedWrappers(); + const sourceLine = "const source = Function.prototype.toString.call(raw);"; + const classTest = "/^\\s*class\\b/.test(source)"; + + assert.equal(abortOnly.split(sourceLine).length - 1, 1); + assert.match(abortOnly, new RegExp(`if \\(!${RegExp.escape(classTest)}\\)`)); + assert.match(hostBindings, /const source = __WdlHostRuntime__\.functionSource\(raw\);/); + assert.match(hostBindings, /if \(__WdlHostRuntime__\.regexpTest\(\/\^\\s\*class\\b\/, source\)\)/); +}); + +test("host wrapper runtime captures platform intrinsics before user module evaluation", () => { + const source = generateHostBindingWrapperModule("worker.js", [], [], ["ROOM"], {}, []); + assert.ok( + source.indexOf(`from "./${HOST_BINDING_RUNTIME_MODULE_NAME}";`) < + source.indexOf('import * as __WdlUserModule__ from "./worker.js";') + ); + assert.match(source, /import \* as __WdlHostRuntime__/); + assert.doesNotMatch(source, /__wdlRunWithRequestContext/); + for (const intrinsic of [ + "Array.prototype.forEach", + "Function.prototype.toString", + "Headers.prototype.get", + "Object.defineProperty", + "Object.entries", + "Object.keys", + "AsyncLocalStorage.prototype.getStore", + "AsyncLocalStorage.prototype.run", + "Reflect.apply", + "Reflect.get", + "RegExp.prototype.test", + 'Object.getOwnPropertyDescriptor(Request.prototype, "headers")', + ]) { + assert.match(HOST_BINDING_RUNTIME_SOURCE, new RegExp(RegExp.escape(intrinsic))); + } + assert.match(HOST_BINDING_RUNTIME_SOURCE, /import \{ sanitizeRequestId \} from "\.\/_wdl-request-id\.js"/); + assert.doesNotMatch(HOST_BINDING_RUNTIME_SOURCE, /\benterWith\b/); + assert.doesNotMatch(source, /Object\.(?:defineProperty|entries|keys)\(/); + assert.doesNotMatch(source, /for \(const .* of /); + assert.doesNotMatch(source, /Function\.prototype\.toString\.call/); +}); + +test("host wrapper request context is invocation-local and preserves return identity", async () => { + const firstStarted = Promise.withResolvers(); + const releaseFirst = Promise.withResolvers(); + const firstResult = hostRequestContext.runWithRequestContext("rid-first", () => { + const result = (async () => { + assert.equal(hostRequestContext.currentRequestId(), "rid-first"); + firstStarted.resolve(undefined); + await releaseFirst.promise; + return hostRequestContext.currentRequestId(); + })(); + assert.equal(hostRequestContext.currentRequestId(), "rid-first"); + return result; + }); + + await firstStarted.promise; + assert.equal(hostRequestContext.currentRequestId(), null); + const secondResult = hostRequestContext.runWithRequestContext("rid-second", async () => { + await Promise.resolve(); + return hostRequestContext.currentRequestId(); + }); + assert.equal(await secondResult, "rid-second"); + releaseFirst.resolve(undefined); + assert.equal(await firstResult, "rid-first"); + assert.equal(hostRequestContext.currentRequestId(), null); + + const nativePromise = Promise.resolve("same"); + assert.equal( + hostRequestContext.runWithRequestContext("rid-identity", () => nativePromise), + nativePromise + ); +}); + +test("host wrapper request context is outermost-only", () => { + const withoutChildId = hostRequestContext.runWithRequestContext("rid-outer", () => + hostRequestContext.runWithRequestContext(null, () => hostRequestContext.currentRequestId())); + const withChildId = hostRequestContext.runWithRequestContext("rid-outer", () => + hostRequestContext.runWithRequestContext("tenant-rid", () => hostRequestContext.currentRequestId())); + const withoutParentId = hostRequestContext.runWithRequestContext(null, () => + hostRequestContext.runWithRequestContext("tenant-rid", () => hostRequestContext.currentRequestId())); + + assert.equal(withoutChildId, "rid-outer"); + assert.equal(withChildId, "rid-outer"); + assert.equal(withoutParentId, null); + assert.equal(hostRequestContext.currentRequestId(), null); +}); + +test("host wrapper request context rejects malformed child request ids", () => { + const inherited = hostRequestContext.runWithRequestContext("rid-outer", () => + hostRequestContext.runWithRequestContext("bad\\id", () => hostRequestContext.currentRequestId())); + + assert.equal(inherited, "rid-outer"); + assert.equal(hostRequestContext.currentRequestId(), null); +}); + +test("host wrapper runtime keeps independently created request-context stores separate", () => { + assert.equal(hostBindingRuntime.currentRequestId, undefined); + assert.equal(hostBindingRuntime.runWithRequestContext, undefined); + const tenantContext = hostBindingRuntime.createRequestContext(); + + const observed = hostRequestContext.runWithRequestContext("rid-parent", () => + tenantContext.runWithRequestContext("tenant-rid", () => hostRequestContext.currentRequestId())); + + assert.equal(observed, "rid-parent"); + assert.equal(hostRequestContext.currentRequestId(), null); +}); + +test("host wrapper request-id extraction ignores tenant-patched request intrinsics", async () => { + const request = new Request("https://worker.example", { + headers: { "x-request-id": "rid-real" }, + }); + await withMockedProperty(Headers.prototype, "get", () => "rid-forged", async () => { + await withMockedPropertyDescriptor(Request.prototype, "headers", { + configurable: true, + get() { + return new Headers({ "x-request-id": "rid-forged" }); + }, + }, async () => { + assert.equal(hostBindingRuntime.requestIdFromEventArg(request), "rid-real"); + }); + }); +}); + +test("generated host wrappers alias legal entrypoint names without declaration collisions", async () => { + const entrypointNames = [ + "user", + "WorkerEntrypoint", + "abortIsolate", + "withRequestContext", + "wrapEnv", + "wrapClassInstance", + "D1Database", + "R2Bucket", + "DurableObjectNamespace", + "Workflow", + ]; + const userUrl = moduleDataUrl(` + ${entrypointNames.map((name) => `export class ${name} {}`).join("\n")} + export default {}; + `); + const cloudflareUrl = moduleDataUrl(` + export class WorkerEntrypoint {} + export function abortIsolate() {} + `); + const d1Url = moduleDataUrl("export class D1Database {}"); + const r2Url = moduleDataUrl("export class R2Bucket {}"); + const doUrl = moduleDataUrl("export class DurableObjectNamespace {}"); + const workflowUrl = moduleDataUrl("export class Workflow {}"); + const source = applyModuleReplacements( + generateHostBindingWrapperModule( + "worker.js", + ["DB"], + ["BUCKET"], + ["ROOM"], + { FLOW: { className: "Workflow" } }, + entrypointNames + ), + [ + ['from "cloudflare:workers"', `from ${JSON.stringify(cloudflareUrl)}`], + [`from "./${HOST_BINDING_RUNTIME_MODULE_NAME}"`, `from ${JSON.stringify(moduleDataUrl(HOST_BINDING_RUNTIME_TEST_SOURCE))}`], + ['from "./_wdl-d1-client.js"', `from ${JSON.stringify(d1Url)}`], + ['from "./_wdl-r2-client.js"', `from ${JSON.stringify(r2Url)}`], + ['from "./_wdl-do-client.js"', `from ${JSON.stringify(doUrl)}`], + ['from "./_wdl-workflows-client.js"', `from ${JSON.stringify(workflowUrl)}`], + ['from "./worker.js"', `from ${JSON.stringify(userUrl)}`], + ] + ); + const wrapped = await import(moduleDataUrl(source)); + const userModule = await import(userUrl); + + for (const name of entrypointNames) { + assert.equal(wrapped[name].name, name); + assert.ok(new wrapped[name]({}, {}) instanceof userModule[name]); + } +}); + +test("host wrapper context uses captured AsyncLocalStorage intrinsics", () => { + const restoreRun = installMockProperty(AsyncLocalStorage.prototype, "run", () => { + throw new Error("live AsyncLocalStorage.run must not run"); + }); + let result; + try { + const requestContext = hostBindingRuntime.createRequestContext(); + result = requestContext.runWithRequestContext("rid-captured", () => { + const restoreGetStore = installMockProperty(AsyncLocalStorage.prototype, "getStore", () => { + throw new Error("live AsyncLocalStorage.getStore must not run"); + }); + try { + return requestContext.currentRequestId(); + } finally { + restoreGetStore(); + } + }); + } finally { + restoreRun(); + } + + assert.equal(result, "rid-captured"); +}); + +test("host wrapper does not inspect or replace tenant return values", () => { + let thenReads = 0; + const value = {}; + Object.defineProperty(value, "then", { + get() { + thenReads += 1; + return undefined; + }, + }); + + const result = hostRequestContext.runWithRequestContext("rid-object", () => value); + + assert.equal(result, value); + assert.equal(thenReads, 0); + assert.equal(hostRequestContext.currentRequestId(), null); +}); diff --git a/tests/unit/s3-retry.test.js b/tests/unit/s3-retry.test.js new file mode 100644 index 0000000..91843a4 --- /dev/null +++ b/tests/unit/s3-retry.test.js @@ -0,0 +1,36 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { + S3_TRANSIENT_RETRIES, + fetchRetryableS3Post, + isTransientS3Response, +} from "../../shared/s3-retry.js"; +import { withMockedProperty } from "../helpers/mock-global.js"; + +test("S3 transient retry policy classifies throttling and server errors", () => { + assert.equal(S3_TRANSIENT_RETRIES, 10); + assert.equal(isTransientS3Response(new Response(null, { status: 429 })), true); + assert.equal(isTransientS3Response(new Response(null, { status: 503 })), true); + assert.equal(isTransientS3Response(new Response(null, { status: 408 })), false); + assert.equal(isTransientS3Response(new Response(null, { status: 400 })), false); +}); + +test("fetchRetryableS3Post retries transport and transient response failures", async () => { + let calls = 0; + const client = { + async fetch() { + calls += 1; + if (calls === 1) throw new Error("transport down"); + if (calls === 2) return new Response("slow down", { status: 500 }); + return new Response("ok"); + }, + }; + + await withMockedProperty(Math, "random", () => 0, async () => { + const response = await fetchRetryableS3Post(client, "https://s3.test/bucket?delete", { + method: "POST", + }); + assert.equal(await response.text(), "ok"); + }); + assert.equal(calls, 3); +}); diff --git a/tests/unit/secret-envelope.test.js b/tests/unit/secret-envelope.test.js index 954f47f..95c3f5c 100644 --- a/tests/unit/secret-envelope.test.js +++ b/tests/unit/secret-envelope.test.js @@ -7,14 +7,16 @@ import { encryptSecretValue, isSecretEnvelope, } from "../../shared/secret-envelope.js"; +import { readRepositoryJson } from "../helpers/load-shared-module.js"; +const SECRET_ENVELOPE_PARITY = readRepositoryJson("tests/fixtures/secret-envelope-parity.json"); const env = { - SECRET_ENVELOPE_LOCAL_KEY_B64: "MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY=", - SECRET_ENVELOPE_KID: "local:test:secret-envelope:v1", + SECRET_ENVELOPE_LOCAL_KEY_B64: SECRET_ENVELOPE_PARITY.provider.localKeyB64, + SECRET_ENVELOPE_KID: SECRET_ENVELOPE_PARITY.provider.kid, }; -function deterministicRandomFactory() { - let next = 0; +function deterministicRandomFactory(start = 0) { + let next = start; return (/** @type {number} */ length) => { const out = new Uint8Array(length); for (let i = 0; i < length; i++) out[i] = next++ & 0xff; @@ -22,70 +24,42 @@ function deterministicRandomFactory() { }; } -test("secret envelope encrypts and decrypts with storage-location AAD", async () => { - const envelope = await encryptSecretValue("sensitive-value", { - env, - hashKey: "secrets:demo:api", - fieldName: "TOKEN", - random: deterministicRandomFactory(), - }); - - assert.equal(isSecretEnvelope(envelope), true); - assert.ok(envelope.startsWith(SECRET_ENVELOPE_PREFIX)); - assert.equal(envelope.includes("sensitive-value"), false); - assert.equal( - await decryptSecretValue(envelope, { - env, - hashKey: "secrets:demo:api", - fieldName: "TOKEN", - }), - "sensitive-value" - ); -}); - -test("secret envelope preserves empty string secrets", async () => { - const envelope = await encryptSecretValue("", { - env, - hashKey: "secrets:demo:api", - fieldName: "EMPTY", - random: deterministicRandomFactory(), - }); - - assert.equal(isSecretEnvelope(envelope), true); - assert.equal( - await decryptSecretValue(envelope, { +test("secret envelope generation and decrypt match the shared parity vectors", async () => { + for (const vector of SECRET_ENVELOPE_PARITY.vectors) { + const envelope = await encryptSecretValue(vector.plaintext, { env, - hashKey: "secrets:demo:api", - fieldName: "EMPTY", - }), - "" - ); + hashKey: vector.hashKey, + fieldName: vector.fieldName, + random: deterministicRandomFactory(vector.randomStart), + }); + + assert.equal(envelope, vector.envelope, vector.name); + assert.equal(isSecretEnvelope(envelope), true, vector.name); + if (vector.plaintext !== "") assert.equal(envelope.includes(vector.plaintext), false, vector.name); + assert.equal( + await decryptSecretValue(envelope, { + env, + hashKey: vector.hashKey, + fieldName: vector.fieldName, + }), + vector.plaintext, + vector.name + ); + } }); -test("secret envelope rejects copy to another hash or field", async () => { - const envelope = await encryptSecretValue("value", { - env, - hashKey: "secrets:demo", - fieldName: "TOKEN", - random: deterministicRandomFactory(), - }); - - await assert.rejects( - decryptSecretValue(envelope, { - env, - hashKey: "secrets:other", - fieldName: "TOKEN", - }), - { code: "secret_decrypt_failed" } - ); - await assert.rejects( - decryptSecretValue(envelope, { - env, - hashKey: "secrets:demo", - fieldName: "OTHER", - }), - { code: "secret_decrypt_failed" } - ); +test("secret envelope rejection behavior matches the shared parity vectors", async () => { + for (const rejection of SECRET_ENVELOPE_PARITY.rejections) { + await assert.rejects( + decryptSecretValue(rejection.envelope, { + env: { ...env, SECRET_ENVELOPE_KID: rejection.configuredKid }, + hashKey: rejection.hashKey, + fieldName: rejection.fieldName, + }), + { code: rejection.jsErrorCode }, + rejection.name + ); + } }); test("secret envelope requires explicit local provider configuration", async () => { @@ -110,24 +84,6 @@ test("secret envelope rejects unprefixed values", async () => { ); }); -test("secret envelope rejects unknown kid", async () => { - const envelope = await encryptSecretValue("value", { - env, - hashKey: "secrets:demo", - fieldName: "TOKEN", - random: deterministicRandomFactory(), - }); - - await assert.rejects( - decryptSecretValue(envelope, { - env: { ...env, SECRET_ENVELOPE_KID: "local:test:secret-envelope:v2" }, - hashKey: "secrets:demo", - fieldName: "TOKEN", - }), - { code: "unknown_kid" } - ); -}); - test("secret envelope reports invalid local provider key as configuration error", async () => { await assert.rejects( encryptSecretValue("value", { diff --git a/tests/unit/shared-primitives.test.js b/tests/unit/shared-primitives.test.js index f21ef84..b5bcfef 100644 --- a/tests/unit/shared-primitives.test.js +++ b/tests/unit/shared-primitives.test.js @@ -19,3 +19,14 @@ test("errorMessage extracts string messages without structured log shape", () => assert.equal(errorMessage(42), "42"); assert.equal(errorMessage(null), "null"); }); + +test("errorMessage never replaces a pathological thrown value", () => { + const throwable = Object.create(null); + assert.equal(errorMessage(throwable), "Unknown error"); + + const brokenError = new Error("original"); + Object.defineProperty(brokenError, "message", { + get() { throw new Error("message getter failed"); }, + }); + assert.equal(errorMessage(brokenError), "Unknown error"); +}); diff --git a/tests/unit/stream-prefix.test.js b/tests/unit/stream-prefix.test.js new file mode 100644 index 0000000..b2fd8d1 --- /dev/null +++ b/tests/unit/stream-prefix.test.js @@ -0,0 +1,25 @@ +import { once } from "node:events"; +import { PassThrough, Writable } from "node:stream"; +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { pipeWithPrefix } from "../../scripts/_stream-prefix.js"; + +test("pipeWithPrefix prefixes complete lines and flushes a partial tail", async () => { + const source = new PassThrough(); + /** @type {string[]} */ + const chunks = []; + const destination = new Writable({ + write(chunk, _encoding, callback) { + chunks.push(String(chunk)); + callback(); + }, + }); + pipeWithPrefix(source, destination, "[task] "); + + source.write("first"); + source.write(" line\nsecond\n"); + source.end("tail"); + await once(source, "end"); + + assert.equal(chunks.join(""), "[task] first line\n[task] second\n[task] tail\n"); +}); diff --git a/tests/unit/style-contracts.test.js b/tests/unit/style-contracts.test.js index 1b1bc65..8fe58c0 100644 --- a/tests/unit/style-contracts.test.js +++ b/tests/unit/style-contracts.test.js @@ -2,9 +2,10 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import path from "node:path"; import { - d1RuntimeServiceRegex, extractAssignedConstant, extractBraceBlock, + extractCapnpConstBlock, + extractCapnpListEntries, extractExportedStringConst, extractRegex, extractStringConst, @@ -13,8 +14,11 @@ import { markdownFiles, objectJsonPayloads, scannedTestFiles, - serviceAnchorRegex, withoutStringAndTemplateLiterals, + yamlAliasCount, + yamlDocument, + yamlDocuments, + yamlMergeAliases, } from "../helpers/style-contract-scanner.js"; import { jsFiles, readRepoFile, rustFiles, sourceFiles, withoutLineComments } from "../helpers/source-scan.js"; @@ -38,6 +42,7 @@ const PRODUCTION_JS_FILES = [ const PLATFORM_HTTP_ERROR_FILES = [ // auth/ is a socket-less JSRPC worker; it returns method payloads instead of // platform HTTP responses. + ...CONTROL_FILES, ...GATEWAY_FILES, ...RUNTIME_FILES, ...D1_RUNTIME_FILES, @@ -52,6 +57,17 @@ const OFFICIAL_DOC_FILES = [ // one-language exception is approved. const INTENTIONAL_ONE_LANGUAGE_ACTIVE_DOCS = new Set(); +function activeBilingualDocFiles() { + const notesDirName = ["arch", "ive"].join(""); + return [ + "README.md", + "README.zh.md", + ...markdownFiles("docs", { ignoreDirs: new Set([notesDirName]) }), + "deploy/kubernetes/README.md", + "deploy/kubernetes/README.zh.md", + ]; +} + // Heuristic multiline scanners for object-literal JSON payload helpers. // They match a first object argument and an optional second object argument // (init/options) without attempting full nested-brace parsing. @@ -247,13 +263,6 @@ test("secret Redis key construction uses the shared JS owner", () => { ); }); -test("do-runtime worker-name grammar matches shared control grammar", () => { - assert.equal( - extractRegex("do-runtime/protocol/wire-grammar.js", "WORKER_NAME_RE"), - extractRegex("shared/ns-pattern.js", "WORKER_NAME_RE") - ); -}); - test("runtime workflow instance id grammar matches shared control grammar", () => { assert.equal( extractRegex("runtime/workflows-client.js", "WORKFLOW_INSTANCE_ID_RE"), @@ -261,23 +270,6 @@ test("runtime workflow instance id grammar matches shared control grammar", () = ); }); -test("runtime workflow dispatch identity grammar mirrors shared grammar", () => { - const shared = readRepoFile("shared/ns-pattern.js"); - assert.equal( - extractRegex("runtime/lib.js", "ROUTE_NS_RE"), - `/^(?:${extractStringConst(shared, "NS_PATTERN")}|__system__)$/` - ); - assert.deepEqual( - extractStringSetConst(readRepoFile("runtime/lib.js"), "RESERVED_TENANT_NS"), - extractStringSetConst(shared, "RESERVED_TENANT_NS") - ); - assert.equal( - extractRegex("runtime/lib.js", "WORKER_NAME_RE"), - extractRegex("shared/ns-pattern.js", "WORKER_NAME_RE") - ); - assert.equal(extractRegex("runtime/lib.js", "VERSION_RE"), "/^v[1-9][0-9]*$/"); -}); - test("D1 object field setters stay shared across wire and transport codecs", () => { assert.match(readRepoFile("shared/d1-data-field.js"), /export function setDataField\(/); for (const file of ["shared/d1-query-wire.js", "shared/d1-transport.js"]) { @@ -312,6 +304,7 @@ test("route/version registry keys go through shared/version.js helpers", () => { // and comments are skipped. const files = [ ...CONTROL_FILES, + ...AUTH_FILES, ...GATEWAY_FILES, ...DO_RUNTIME_FILES, ...RUNTIME_FILES, @@ -327,6 +320,18 @@ test("route/version registry keys go through shared/version.js helpers", () => { if (/`worker:do-storage:\$\{/.test(source)) { offenders.push(`${file}: inline worker:do-storage: — use doStorageIdKey(ns, worker)`); } + if (/`hosts:\$\{/.test(source)) offenders.push(`${file}: inline hosts: — use hostsKey(ns)`); + if (/`ns-hosts:\$\{/.test(source)) offenders.push(`${file}: inline ns-hosts: — use nsHostsKey(ns)`); + if (/`worker:\$\{[^`]*:next_version`/.test(source)) { + offenders.push(`${file}: inline next_version counter — use nextVersionKey(ns, worker)`); + } + if (/"namespaces"/.test(source)) offenders.push(`${file}: inline namespaces set — use NAMESPACES_KEY`); + if (/"declared-hosts"/.test(source)) { + offenders.push(`${file}: inline declared-hosts set — use DECLARED_HOSTS_KEY`); + } + if (/"host-declarations:"/.test(source)) { + offenders.push(`${file}: inline host-declarations prefix — use hostDeclarationsKey(host)`); + } } // Rust services are production readers too; rust/common/version.rs holds the // only literals (mirror of shared/version.js), every other crate must call @@ -347,6 +352,21 @@ test("route/version registry keys go through shared/version.js helpers", () => { assert.deepEqual(offenders, [], `route/version key literals must use shared helpers:\n${offenders.join("\n")}`); }); +test("platform domain configuration uses the shared normalized owner", () => { + const owner = withoutLineComments(readRepoFile("shared/ns-pattern.js")); + assert.match(owner, /DEFAULT_PLATFORM_DOMAIN = "workers\.local"/); + for (const file of [ + "control/handlers/deploy.js", + "control/handlers/promote.js", + "control/handlers/hosts.js", + "gateway/index.js", + ]) { + const source = withoutLineComments(readRepoFile(file)); + assert.match(source, /\bplatformDomainFromEnv\b/, file); + assert.doesNotMatch(source, /"workers\.local"/, file); + } +}); + test("secret Redis key literals stay aligned across JS and redis-proxy", () => { const js = readRepoFile("shared/secret-keys.js"); const rust = readRepoFile("rust/redis-proxy/src/runtime.rs"); @@ -357,42 +377,22 @@ test("secret Redis key literals stay aligned across JS and redis-proxy", () => { assert.match(rust, /format!\("secrets:\{\}:\{\}", q\.ns, q\.worker\)/); }); -test("test bundleKey stubs stay production-faithful", () => { - const offenders = []; - const bundleKeyStubBodies = (/** @type {string} */ source) => { - const bodies = []; - for (const match of source.matchAll(/export function bundleKey\(|const bundleKey\s*=/g)) { - const start = match.index ?? 0; - const nextExport = source.indexOf("\nexport function ", start + 1); - const nextConst = source.indexOf("\nconst ", start + 1); - const nextBoundary = [nextExport, nextConst] - .filter((idx) => idx !== -1) - .sort((a, b) => a - b)[0] ?? source.length; - bodies.push(source.slice(start, nextBoundary)); - } - return bodies; - }; - for (const file of jsFiles("tests/unit")) { - const source = withoutLineComments(readRepoFile(file)); - for (const body of bundleKeyStubBodies(source)) { - const hasVersionSegmentComposition = - /":v:"\s*\+\s*n/.test(body) || /:v:\$\{?\s*n\s*\}?/.test(body); - if (!/\bparseVersion\(version\)/.test(body) || !hasVersionSegmentComposition) { - offenders.push(`${file}: bundleKey stub must parse vN and compose worker:::v:`); - } - } - } - assert.deepEqual(offenders, []); -}); - test("worker delete lock key stays aligned across control and workflows", () => { - const controlLib = withoutLineComments(readRepoFile("control/lib.js")); + const sharedVersion = withoutLineComments(readRepoFile("shared/version.js")); + const commonVersion = withoutLineComments(readRepoFile("rust/common/src/version.rs")); const controlShared = withoutLineComments(readRepoFile("control/shared.js")); const workflowsActiveExport = withoutLineComments(readRepoFile("rust/workflows/src/api/active_export.rs")); - assert.match(controlLib, /`worker-delete-lock:\$\{ns\}:\$\{worker\}`/); + assert.match(sharedVersion, /`worker-delete-lock:\$\{ns\}:\$\{worker\}`/); + assert.match(commonVersion, /format!\("worker-delete-lock:\{ns\}:\{worker\}"\)/); assert.match(controlShared, /\bdeleteLockKey\(ns, worker\)/); - assert.match(workflowsActiveExport, /format!\("worker-delete-lock:\{ns\}:\{worker\}"\)/); + assert.match(workflowsActiveExport, /\bworker_delete_lock_key\(ns, worker\)/); + + const rustOwner = "rust/common/src/version.rs"; + const offenders = rustFiles("rust").filter((file) => + file !== rustOwner && withoutLineComments(readRepoFile(file)).includes("worker-delete-lock:") + ); + assert.deepEqual(offenders, []); }); test("product success payloads use camelCase fields", () => { @@ -895,36 +895,20 @@ test("Redis command metric allow-list covers shared Redis wrappers", () => { ); }); -test("route invalidation channel literals stay aligned across control and gateway", () => { - const shared = withoutLineComments(readRepoFile("control/shared.js")); - const routing = withoutLineComments(readRepoFile("control/routing.js")); - const gateway = withoutLineComments(readRepoFile("gateway/runtime.js")); - const routesChannel = extractStringConst(shared, "ROUTES_CHANNEL"); - const routesFlushChannel = extractStringConst(shared, "ROUTES_FLUSH_CHANNEL"); - const patternsChannel = extractStringConst(shared, "PATTERNS_CHANNEL"); - - assert.equal(extractStringConst(routing, "ROUTES_CHANNEL"), routesChannel); - assert.equal(extractStringConst(routing, "PATTERNS_CHANNEL"), patternsChannel); - assert.match( - gateway, - new RegExp(`\\[\\s*"${RegExp.escape(routesChannel)}",\\s*"${RegExp.escape(routesFlushChannel)}",\\s*"${RegExp.escape(patternsChannel)}"\\s*\\]`) - ); - assert.match(gateway, new RegExp(`channel === "${RegExp.escape(routesFlushChannel)}"`)); - assert.match(gateway, new RegExp(`channel === "${RegExp.escape(patternsChannel)}"`)); -}); - test("declared-host Redis key literals stay aligned across control, gateway, and docs", () => { + const version = withoutLineComments(readRepoFile("shared/version.js")); const shared = withoutLineComments(readRepoFile("control/shared.js")); const routing = withoutLineComments(readRepoFile("control/routing.js")); const gateway = withoutLineComments(readRepoFile("gateway/runtime.js")); const layoutEn = readRepoFile("docs/redis-key-layout.md"); const layoutZh = readRepoFile("docs/redis-key-layout.zh.md"); - const declaredHostsKey = extractStringConst(shared, "DECLARED_HOSTS_KEY"); - const hostDeclarationsPrefix = extractStringConst(shared, "HOST_DECLARATIONS_PREFIX"); + const declaredHostsKey = extractStringConst(version, "DECLARED_HOSTS_KEY"); + const hostDeclarationsPrefix = extractStringConst(version, "HOST_DECLARATIONS_PREFIX"); assert.match(routing, new RegExp(`\\bDECLARED_HOSTS_KEY\\b`)); - assert.match(routing, new RegExp(`\\bHOST_DECLARATIONS_PREFIX\\b`)); - assert.equal(extractStringConst(gateway, "DECLARED_HOSTS_KEY"), declaredHostsKey); + assert.match(routing, /\bhostDeclarationsKey\b/); + assert.match(shared, /\bhostDeclarationsKey\b/); + assert.match(gateway, /\bDECLARED_HOSTS_KEY\b/); for (const source of [layoutEn, layoutZh]) { assert.match(source, new RegExp(`\\b${RegExp.escape(declaredHostsKey)}\\b`)); assert.match(source, new RegExp(`\\b${RegExp.escape(hostDeclarationsPrefix)}`)); @@ -932,12 +916,13 @@ test("declared-host Redis key literals stay aligned across control, gateway, and }); test("gateway strips every client-supplied platform header it injects", () => { - const source = withoutLineComments(readRepoFile("gateway/index.js")); - const headerList = /const INTERNAL_FORWARD_HEADERS = \[([\s\S]*?)\];/.exec(source); + const gateway = withoutLineComments(readRepoFile("gateway/index.js")); + const owner = withoutLineComments(readRepoFile("gateway/lib.js")); + const headerList = /const INTERNAL_FORWARD_HEADERS = \[([\s\S]*?)\];/.exec(owner); assert.ok(headerList, "gateway INTERNAL_FORWARD_HEADERS list must be present"); const stripped = new Set([...headerList[1].matchAll(/"([^"]+)"/g)].map((match) => match[1])); const injectedNonPrefixedInternal = new Set( - [...source.matchAll(/forwardRequest\.headers\.set\("(x-worker[^"]+)"/g)] + [...gateway.matchAll(/forwardRequest\.headers\.set\("(x-worker[^"]+)"/g)] .map((match) => match[1]) ); const missing = [...injectedNonPrefixedInternal].filter((header) => !stripped.has(header)).toSorted(); @@ -946,22 +931,13 @@ test("gateway strips every client-supplied platform header it injects", () => { [], `gateway INTERNAL_FORWARD_HEADERS must include injected non-prefixed internal headers: ${missing.join(", ")}` ); - assert.match(source, /const INTERNAL_HEADER_PREFIX = "x-wdl-"/); - assert.match(source, /header\.toLowerCase\(\)\.startsWith\(INTERNAL_HEADER_PREFIX\)/); - assert.match(source, /headers\.delete\(header\)/); + assert.match(owner, /const INTERNAL_HEADER_PREFIX = "x-wdl-"/); + assert.match(owner, /name\.toLowerCase\(\)\.startsWith\(INTERNAL_HEADER_PREFIX\)/); + assert.match(owner, /headers\.delete\(name\)/); + assert.match(gateway, /deleteGatewayInternalHeaders\(forwardRequest\.headers\)/); const websocket = withoutLineComments(readRepoFile("gateway/websocket.js")); - const websocketHeaderList = /const INTERNAL_FORWARD_HEADERS = \[([\s\S]*?)\];/.exec(websocket); - assert.ok(websocketHeaderList, "gateway websocket INTERNAL_FORWARD_HEADERS list must be present"); - const websocketStripped = new Set([...websocketHeaderList[1].matchAll(/"([^"]+)"/g)].map((match) => match[1])); - assert.deepEqual( - ["x-worker-id", "x-worker-prefix"].filter((header) => !websocketStripped.has(header)), - [], - "gateway websocket must strip non-prefixed internal headers from upgrade responses" - ); - assert.match(websocket, /const INTERNAL_HEADER_PREFIX = "x-wdl-"/); - assert.match(websocket, /name\.toLowerCase\(\)\.startsWith\(INTERNAL_HEADER_PREFIX\)/); - assert.match(websocket, /out\.delete\(name\)/); + assert.match(websocket, /deleteGatewayInternalHeaders\(out\)/); }); test("D1 owner protocol errors stay aligned with runtime stale-hint handling", () => { @@ -1150,10 +1126,12 @@ test("redis-proxy HTTP observability matches platform request strategy", () => { assert.match(lib, /struct ResponseError/); }); -test("embedded DO alarm shim follows shared log stream routing", () => { +test("embedded DO alarm shim keeps best-effort log routing no-throw", () => { const source = withoutLineComments(readRepoFile("do-runtime/alarm-shim-source.js")); + const logger = /function logStructured\([^]*?\n}\n\nfunction ensureAlarmTable/.exec(source)?.[0] || ""; assert.equal(/console\.warn\(/.test(source), false); - assert.match(source, /if \(level === "error"\) console\.error\(line\);\n {2}else console\.log\(line\);/); + assert.match(logger, /if \(level === "error"\) console\.error\(line\);\s+else console\.log\(line\);/); + assert.match(logger, /try \{[^]*} catch \{/); }); test("active docs and sources do not point at note paths", () => { @@ -1181,12 +1159,7 @@ test("active docs and sources do not point at note paths", () => { }); test("active bilingual docs stay paired", () => { - const notesDirName = ["arch", "ive"].join(""); - const files = [ - "README.md", - "README.zh.md", - ...markdownFiles("docs", { ignoreDirs: new Set([notesDirName]) }), - ]; + const files = activeBilingualDocFiles(); const fileSet = new Set(files); const offenders = []; for (const file of files) { @@ -1202,6 +1175,50 @@ test("active bilingual docs stay paired", () => { assert.deepEqual(offenders.toSorted(), []); }); +test("owning bilingual docs retain selected cross-tier protocol literals", () => { + // Explicitly pin only high-value literals in their owning docs. Do not return + // to scanning every uppercase token, example port, or incidental DB mention. + /** @param {string} value */ + const codeSpan = (value) => `\`${value}\``; + /** @type {Array<{ files: string[], literals: string[] }>} */ + const contracts = [ + { + files: ["docs/modules/infra.md", "docs/modules/infra.zh.md"], + literals: [ + "WDL_INTERNAL_AUTH_TOKEN", + "x-wdl-internal-auth", + ":8088", + ":8787", + ":8788", + ":9120", + ], + }, + { + files: ["docs/redis-key-layout.md", "docs/redis-key-layout.zh.md"], + literals: ["DB 0", "DB 1", "DB 2"], + }, + { + files: ["docs/modules/control-auth.md", "docs/modules/control-auth.zh.md"], + literals: ["X-Admin-Token"], + }, + ]; + for (const suffix of [".v2", "+v2", "~v2", "-v2"]) { + assert.equal(codeSpan(`x-wdl-internal-auth${suffix}`).includes(codeSpan("x-wdl-internal-auth")), false); + } + assert.equal(codeSpan(":8787abc").includes(codeSpan(":8787")), false); + assert.equal(codeSpan("DB 01").includes(codeSpan("DB 0")), false); + const offenders = []; + for (const { files, literals } of contracts) { + for (const file of files) { + const source = readRepoFile(file); + for (const literal of literals) { + if (!source.includes(codeSpan(literal))) offenders.push(`${file}: missing ${literal}`); + } + } + } + assert.deepEqual(offenders, []); +}); + test("protocol and contributor contracts stay discoverable from active docs", () => { const links = [ ["CLAUDE.md", "docs/protocol-contracts.md"], @@ -1337,7 +1354,8 @@ test("do-runtime operational logs avoid known camelCase field drift", () => { test("DO alarm tokens stay generated inside the storage shim", () => { const source = withoutLineComments(readRepoFile("do-runtime/alarm-shim-source.js")); - assert.match(source, /function alarmToken\(\)\s*\{\s*return crypto\.randomUUID\(\);\s*\}/); + assert.match(source, /const cryptoRandomUUID = crypto\.randomUUID;/); + assert.match(source, /function alarmToken\(\)\s*\{\s*return reflectApply\(cryptoRandomUUID, nativeCrypto, \[\]\);\s*\}/); assert.match(source, /const token = alarmToken\(\);/); assert.match(source, /alarmBinding\.setAlarmIndex\(\{\s*className,\s*objectName,\s*scheduledTime: alarmTime,\s*retryCount: 0,\s*token,/); }); @@ -1438,26 +1456,189 @@ test("owner forward metrics classify non-error HTTP statuses consistently", () = }); test("local compose services inherit shared image anchors", () => { - const source = readRepoFile("docker-compose.yml"); - const offenders = []; - for (const service of ["redis-proxy-user", "redis-proxy-system", "scheduler"]) { - if (!serviceAnchorRegex(service, "rust-sidecar-image").test(source)) { - offenders.push(service); - } + const document = yamlDocument("docker-compose.yml"); + const merged = /** @type {{ services: Record }} */ ( + yamlDocument("docker-compose.yml", { merge: true }).toJS() + ); + assert.deepEqual(yamlMergeAliases(document, ["x-redis-proxy-service"]), ["rust-sidecar-image"]); + assert.deepEqual(yamlMergeAliases(document, ["x-d1-runtime-service"]), ["workerd-image"]); + assert.deepEqual(yamlMergeAliases(document, ["x-do-runtime-service"]), ["workerd-image"]); + assert.deepEqual( + yamlMergeAliases(document, ["x-redis-proxy-service", "environment"]), + ["internal-auth-env", "secret-envelope-env"] + ); + assert.deepEqual( + yamlMergeAliases(document, ["services", "system-runtime", "environment"]), + ["internal-auth-env", "secret-envelope-env"] + ); + assert.equal(yamlAliasCount(document, "secret-envelope-env"), 2); + + /** @type {Map} */ + const serviceAnchors = new Map(); + for (const service of ["redis-proxy-user", "redis-proxy-system", "redis-proxy-do"]) { + serviceAnchors.set(service, "redis-proxy-service"); } + for (const service of ["scheduler", "workflows"]) serviceAnchors.set(service, "rust-sidecar-image"); for (const service of ["user-runtime", "system-runtime", "gateway"]) { - if (!serviceAnchorRegex(service, "workerd-image").test(source)) { - offenders.push(service); - } + serviceAnchors.set(service, "workerd-image"); } - // Strict by design: d1-multi profile filters stay before anchor inheritance - // so profile-only variants remain visually distinct from the default task. for (const service of ["d1-runtime", "d1-runtime-a", "d1-runtime-b", "d1-runtime-c"]) { - if (!d1RuntimeServiceRegex(service).test(source)) { - offenders.push(service); + serviceAnchors.set(service, "d1-runtime-service"); + } + for (const service of ["do-runtime", "do-runtime-a", "do-runtime-b", "do-runtime-c"]) { + serviceAnchors.set(service, "do-runtime-service"); + } + for (const [service, anchor] of serviceAnchors) { + assert.deepEqual(yamlMergeAliases(document, ["services", service]), [anchor], service); + } + for (const family of ["d1", "do"]) { + assert.equal(merged.services[`${family}-runtime`].profiles, undefined); + for (const suffix of ["a", "b", "c"]) { + assert.deepEqual(merged.services[`${family}-runtime-${suffix}`].profiles, [`${family}-multi`]); } } - assert.deepEqual(offenders, []); +}); + +test("published Compose overrides cover every locally built image service", () => { + const local = /** @type {{ + * services: Record, + * }} */ (yamlDocument("docker-compose.yml", { merge: true }).toJS()); + const publishedDocument = yamlDocument("docker-compose.images.yml"); + const published = /** @type {{ + * services: Record, + * }} */ (yamlDocument("docker-compose.images.yml", { merge: true }).toJS()); + const familyByDockerfile = new Map([ + ["Dockerfile.rust", { alias: "rust-published", image: "docker.io/getwdl/wdl-rust:latest" }], + ["Dockerfile.workerd", { alias: "workerd-published", image: "docker.io/getwdl/wdl-workerd:latest" }], + ]); + for (const family of familyByDockerfile.values()) { + const build = /** @type {{ tag?: string, value?: unknown } | undefined} */ ( + publishedDocument.getIn([`x-${family.alias}`, "build"], true) + ); + assert.equal(build?.tag, "!reset", `${family.alias} must reset inherited build config`); + assert.equal(build?.value, "null", `${family.alias} build reset must use null`); + } + const expected = new Map(); + for (const [service, config] of Object.entries(local.services)) { + if (!config.build) continue; + assert.equal(config.build.context, ".", `${service} build context`); + const family = familyByDockerfile.get(config.build.dockerfile || ""); + assert.ok(family, `${service} must use a recognized shared Dockerfile`); + expected.set(service, family); + } + + assert.deepEqual(Object.keys(published.services).toSorted(), [...expected.keys()].toSorted()); + for (const [service, family] of expected) { + assert.deepEqual(yamlMergeAliases(publishedDocument, ["services", service]), [family.alias], service); + assert.deepEqual(published.services[service], { + image: family.image, + pull_policy: "always", + build: "null", + }); + } +}); + +test("Kubernetes runtime families pin the redis-proxy functional contract", () => { + const files = [ + "deploy/kubernetes/base/user-runtime.yaml", + "deploy/kubernetes/base/system-runtime.yaml", + "deploy/kubernetes/base/do-runtime.yaml", + ]; + const sidecars = files.map((file) => { + const resources = /** @type {Array<{ + * kind?: string, + * spec?: { template?: { spec?: { containers?: Array> } } }, + * }>} */ (yamlDocuments(file).map((document) => document.toJS())); + const workload = resources.find((resource) => + resource.kind === "Deployment" || resource.kind === "StatefulSet" + ); + assert.ok(workload, `${file} must contain a Deployment or StatefulSet`); + const sidecar = workload.spec?.template?.spec?.containers + ?.find((container) => container.name === "redis-proxy"); + assert.ok(sidecar, `${file} must contain the redis-proxy sidecar`); + return sidecar; + }); + + for (let index = 0; index < sidecars.length; index += 1) { + const sidecar = sidecars[index]; + assert.equal(sidecar.image, "docker.io/getwdl/wdl-rust:latest", `${files[index]} image`); + assert.deepEqual(sidecar.command, ["/redis-proxy"], `${files[index]} command`); + assert.deepEqual( + /** @type {Array> | undefined} */ (sidecar.ports) + ?.find((port) => port.name === "redis-proxy"), + { name: "redis-proxy", containerPort: 7070 }, + `${files[index]} port` + ); + assert.deepEqual(sidecar.envFrom, [ + { configMapRef: { name: "wdl-config" } }, + { secretRef: { name: "wdl-secrets" } }, + ], `${files[index]} envFrom`); + for (const probe of ["readinessProbe", "livenessProbe"]) { + assert.deepEqual(sidecar[probe], { + exec: { command: ["/redis-proxy", "healthcheck"] }, + }, `${files[index]} ${probe}`); + } + const resources = /** @type {{ requests?: Record, limits?: Record } | undefined} */ ( + sidecar.resources + ); + assert.ok(resources?.requests?.cpu, `${files[index]} resource request cpu`); + assert.ok(resources?.requests?.memory, `${files[index]} resource request memory`); + assert.ok(resources?.limits?.memory, `${files[index]} resource limit memory`); + } +}); + +test("Kubernetes NetworkPolicies pin the per-component ingress matrix", () => { + const documents = yamlDocuments("deploy/kubernetes/base/network-policy.yaml"); + /** @type {Record} */ + const actual = {}; + for (const document of documents) { + const policy = /** @type {{ + * spec?: { + * podSelector?: { matchLabels?: Record }, + * ingress?: Array<{ + * from?: Array<{ podSelector?: { matchLabels?: Record } }>, + * ports?: Array<{ port?: unknown }>, + * }>, + * }, + * }} */ (document.toJS()); + const receiver = policy?.spec?.podSelector?.matchLabels?.["app.kubernetes.io/component"]; + if (typeof receiver !== "string") continue; + assert.equal(actual[receiver], undefined, `duplicate NetworkPolicy receiver ${receiver}`); + + const edges = []; + for (const ingress of policy.spec?.ingress || []) { + const rawCallers = ingress.from === undefined + ? ["*"] + : ingress.from.map((peer) => peer?.podSelector?.matchLabels?.["app.kubernetes.io/component"]); + const ports = (ingress.ports || []).map((entry) => Number(entry.port)); + assert.ok(rawCallers.every((caller) => typeof caller === "string"), `${receiver} callers must use component selectors`); + assert.ok(ports.every(Number.isInteger), `${receiver} ingress ports must be integers`); + const callers = /** @type {string[]} */ (rawCallers); + for (const caller of callers) { + for (const port of ports) edges.push(`${caller}:${port}`); + } + } + actual[receiver] = edges.toSorted(); + } + + assert.deepEqual(actual, { + gateway: ["*:8080"], + "user-runtime": ["gateway:8081", "scheduler:8088", "workflows:8088"], + "system-runtime": ["gateway:8081", "gateway:8082", "scheduler:8088", "workflows:8088"], + "d1-runtime": ["d1-runtime:8787", "do-runtime:8787", "system-runtime:8787", "user-runtime:8787"], + "do-runtime": ["do-runtime:8788", "system-runtime:8788", "user-runtime:8788", "workflows:8788"], + workflows: ["do-runtime:9120", "scheduler:9120", "system-runtime:9120", "user-runtime:9120"], + valkey: [ + "d1-runtime:6379", + "do-runtime:6379", + "gateway:6379", + "scheduler:6379", + "system-runtime:6379", + "user-runtime:6379", + "workflows:6379", + ], + s3mock: ["do-runtime:9090", "system-runtime:9090", "user-runtime:9090"], + }); }); test("local compose routes private HTTP hops through Envoy only", () => { @@ -1626,6 +1807,126 @@ test("local compose routes private HTTP hops through Envoy only", () => { assert.ok((envoy.match(/preserve_external_request_id: true/g) || []).length >= 5); }); +test("user and system runtime worker contracts differ only at privilege fields", () => { + const user = readRepoFile("runtime/config-user.capnp"); + const system = readRepoFile("runtime/config-system.capnp"); + + const entryMap = (/** @type {string[]} */ entries) => Object.fromEntries( + entries.map((entry) => { + const name = /\(name\s*=\s*"([^"]+)"/.exec(entry); + assert.ok(name, `Cap'n Proto tuple must have a name: ${entry}`); + return [name[1], entry]; + }).toSorted(([left], [right]) => left.localeCompare(right)) + ); + const stringField = (/** @type {string} */ block, /** @type {string} */ field) => { + const match = new RegExp(`\\b${RegExp.escape(field)}\\s*=\\s*"([^"]+)"`).exec(block); + assert.ok(match, `Cap'n Proto field ${field} must be present`); + return match[1]; + }; + const listField = (/** @type {string} */ block, /** @type {string} */ field) => { + const match = new RegExp(`\\b${RegExp.escape(field)}\\s*=\\s*(\\[[^\\]]*\\])`).exec(block); + assert.ok(match, `Cap'n Proto field ${field} must be present`); + return match[1].replace(/\s+/g, ""); + }; + const withoutEntries = (/** @type {Record} */ entries, /** @type {string[]} */ names) => Object.fromEntries( + Object.entries(entries).filter(([name]) => !names.includes(name)) + ); + + const userLoader = extractCapnpConstBlock(user, "loaderWorker"); + const systemLoader = extractCapnpConstBlock(system, "loaderWorker"); + assert.deepEqual( + entryMap(extractCapnpListEntries(userLoader, "modules")), + entryMap(extractCapnpListEntries(systemLoader, "modules")) + ); + assert.equal(stringField(userLoader, "compatibilityDate"), stringField(systemLoader, "compatibilityDate")); + assert.equal(listField(userLoader, "compatibilityFlags"), listField(systemLoader, "compatibilityFlags")); + + const userBindings = entryMap(extractCapnpListEntries(userLoader, "bindings")); + const systemBindings = entryMap(extractCapnpListEntries(systemLoader, "bindings")); + assert.deepEqual( + withoutEntries(userBindings, ["SERVICE_NAME", "PUBLIC_NETWORK"]), + withoutEntries(systemBindings, ["SERVICE_NAME", "PUBLIC_NETWORK"]) + ); + assert.match(userBindings.SERVICE_NAME, /text = "user-runtime"/); + assert.match(systemBindings.SERVICE_NAME, /text = "system-runtime"/); + assert.match(userBindings.PUBLIC_NETWORK, /service = "public-network"/); + assert.match(systemBindings.PUBLIC_NETWORK, /service = "network"/); + assert.equal(stringField(userLoader, "globalOutbound"), "internal-network"); + assert.equal(stringField(systemLoader, "globalOutbound"), "network"); + + const userOwner = extractCapnpConstBlock(user, "doOwnerNetworkWorker"); + const systemOwner = extractCapnpConstBlock(system, "doOwnerNetworkWorker"); + assert.deepEqual( + entryMap(extractCapnpListEntries(userOwner, "modules")), + entryMap(extractCapnpListEntries(systemOwner, "modules")) + ); + assert.deepEqual( + entryMap(extractCapnpListEntries(userOwner, "bindings")), + entryMap(extractCapnpListEntries(systemOwner, "bindings")) + ); + assert.equal(stringField(userOwner, "compatibilityDate"), stringField(systemOwner, "compatibilityDate")); + assert.equal(stringField(userOwner, "globalOutbound"), "internal-network"); + assert.equal(stringField(systemOwner, "globalOutbound"), "network"); + + const userTail = extractCapnpConstBlock(user, "tailWorker"); + const systemTail = extractCapnpConstBlock(system, "tailWorker"); + assert.deepEqual( + entryMap(extractCapnpListEntries(userTail, "modules")), + entryMap(extractCapnpListEntries(systemTail, "modules")) + ); + const userTailBindings = entryMap(extractCapnpListEntries(userTail, "bindings")); + const systemTailBindings = entryMap(extractCapnpListEntries(systemTail, "bindings")); + assert.deepEqual( + withoutEntries(userTailBindings, ["SERVICE_NAME"]), + withoutEntries(systemTailBindings, ["SERVICE_NAME"]) + ); + assert.match(userTailBindings.SERVICE_NAME, /text = "user-runtime-tail"/); + assert.match(systemTailBindings.SERVICE_NAME, /text = "system-runtime-tail"/); + assert.equal(stringField(userTail, "compatibilityDate"), stringField(systemTail, "compatibilityDate")); + assert.equal(stringField(userTail, "globalOutbound"), "internal-network"); + assert.equal(stringField(systemTail, "globalOutbound"), "network"); +}); + +test("local workerd configs preserve base service and socket topology", () => { + const pairs = [ + ["gateway/config.capnp", "gateway/config-local.capnp"], + ["runtime/config-user.capnp", "runtime/config-user-local.capnp"], + ["runtime/config-system.capnp", "runtime/config-system-local.capnp"], + ["do-runtime/config.capnp", "do-runtime/config-local.capnp"], + ]; + const entryMap = (/** @type {string[]} */ entries) => Object.fromEntries( + entries.map((entry) => { + const name = /\(name\s*=\s*"([^"]+)"/.exec(entry); + assert.ok(name, `Cap'n Proto tuple must have a name: ${entry}`); + return [name[1], entry]; + }).toSorted(([left], [right]) => left.localeCompare(right)) + ); + const normalizeBaseReference = (/** @type {string} */ entry) => entry.replace(/\.Base\./g, "."); + + for (const [baseFile, localFile] of pairs) { + const base = extractCapnpConstBlock(readRepoFile(baseFile), "config"); + const local = extractCapnpConstBlock(readRepoFile(localFile), "config"); + const baseServices = entryMap(extractCapnpListEntries(base, "services")); + const localServices = entryMap(extractCapnpListEntries(local, "services")); + assert.deepEqual(Object.keys(localServices), Object.keys(baseServices), `${localFile} service names`); + + for (const [name, baseService] of Object.entries(baseServices)) { + const localService = localServices[name]; + if (baseService.includes("external =")) { + assert.match(localService, /external = \(address = "[^"]+", http = \(\)\)/, `${localFile}:${name}`); + } else { + assert.equal(normalizeBaseReference(localService), baseService, `${localFile}:${name}`); + } + } + + assert.deepEqual( + entryMap(extractCapnpListEntries(local, "sockets")), + entryMap(extractCapnpListEntries(base, "sockets")), + `${localFile} sockets` + ); + } +}); + test("S3 query encoding stays aligned between shared and injected runtime helpers", () => { const extract = (/** @type {string} */ file) => { const source = readRepoFile(file); @@ -1726,6 +2027,7 @@ test("workflow instance state is owned by workflows DB2", () => { "wf:by-worker:", "wf:by-workflow:", "wf:by-version:", + "wf:pending-version:", "wf:retention", ]; const allowed = new Set([ @@ -1733,9 +2035,9 @@ test("workflow instance state is owned by workflows DB2", () => { "tests/integration/workflows-runtime-core.test.js", "tests/integration/workflows-runtime-scheduler.test.js", "tests/integration/workflows-runtime-pausing.test.js", - "tests/integration/workflows-runtime-retention.test.js", "tests/unit/style-contracts.test.js", "rust/workflows/src/keys.rs", + "rust/workflows/src/schema.rs", "rust/workflows/src/tests.rs", ]); const allowedFilePrefixes = new Set([ @@ -1755,6 +2057,7 @@ test("workflow instance state is owned by workflows DB2", () => { ...jsFiles("do-runtime"), ...jsFiles("tests/unit"), ...jsFiles("tests/integration"), + ...rustFiles("rust"), ]; const offenders = []; for (const file of files) { @@ -1820,7 +2123,7 @@ test("cron discovery index literals stay aligned across scheduler and control", assert.ok(control.includes(workerIndex), `control cron indexes must contain ${workerIndex}`); assert.ok(scheduler.includes(workerIndex), `scheduler cron indexes must contain ${workerIndex}`); assert.ok(integration.includes(workerIndex), `cron integration tests must contain ${workerIndex}`); - assert.ok(routing.includes("stageCronSlotRef"), "control routing should stage cron refs through the shared helper"); + assert.ok(routing.includes("stageCronProjection"), "control routing should stage the scheduler projection through its owner"); assert.equal(/`crons:\$\{/.test(routing), false, "control routing should not hand-roll cron hash keys"); assert.equal(/`cron-slot:\$\{/.test(routing), false, "control routing should not hand-roll cron slot keys"); @@ -1924,26 +2227,11 @@ test("D1 and DO workerd env tunables are exposed through capnp bindings", () => "DO_OWNER_LEASE_GUARD_MS", "DO_RENEW_CONCURRENCY", "DO_DRAIN_IN_FLIGHT_TIMEOUT_MS", - "DO_TEST_HOOKS", ]) { assert.equal(exposed(doRuntime, name), true, `${name} must reach do-runtime workerd env`); } }); -test("Terraform gates DO test hooks like D1 test hooks", () => { - const rootVars = readRepoFile("terraform/variables.tf"); - const moduleVars = readRepoFile("terraform/modules/compute/variables.tf"); - const main = readRepoFile("terraform/main.tf"); - const doService = readRepoFile("terraform/modules/compute/do_runtime_service.tf"); - - assert.match(rootVars, /variable "do_test_hooks_enabled"/); - assert.match(moduleVars, /variable "do_test_hooks_enabled"/); - assert.match(main, /do_test_hooks_enabled\s+=\s+var\.do_test_hooks_enabled/); - assert.match(doService, /var\.do_test_hooks_enabled \? \[/); - assert.match(doService, /\{ name = "DO_TEST_HOOKS", value = "1" \}/); - assert.match(doService, /!var\.do_test_hooks_enabled \|\| can\(regex\("\(\^\|-\)test\(\$\|-\)", var\.name\)\)/); -}); - test("D1 and DO workerd containers keep explicit memory ceilings", () => { const rootVars = readRepoFile("terraform/variables.tf"); const moduleVars = readRepoFile("terraform/modules/compute/variables.tf"); @@ -2135,30 +2423,6 @@ test("Terraform ECS services use Fargate-only launch contracts", () => { } }); -test("DO RPC JSON-data validators stay aligned across client and server", () => { - // Match jsonDataError() through the following public checker declaration. - // The checker may be preceded by JSDoc and may be exported assertJsonRpcArgs() - // or local requireJsonData(). - const jsonDataErrorWithChecker = - /function jsonDataError\([^]*?\n}\n\n(?:\/\*\*[^]*?\*\/\n)?(?:export function assertJsonRpcArgs|function requireJsonData)/; - const trailingCheckerMarker = - /\n\n(?:\/\*\*[^]*?\*\/\n)?(?:export function assertJsonRpcArgs|function requireJsonData)$/; - const extract = (/** @type {string} */ file) => { - const source = readRepoFile(file); - const match = source.match(jsonDataErrorWithChecker); - assert.ok(match, `${file} must define jsonDataError next to its public checker`); - return match[0] - .replace(trailingCheckerMarker, "") - .replace(/\s+/g, " ") - .trim(); - }; - - assert.equal( - extract("runtime/_wdl-do-transport.js"), - extract("do-runtime/protocol.js") - ); -}); - test("DO invoke request-size caps stay aligned across client and server", () => { const runtime = readRepoFile("runtime/_wdl-do-transport.js"); const protocol = readRepoFile("do-runtime/protocol.js"); @@ -2171,12 +2435,10 @@ test("DO invoke request-size caps stay aligned across client and server", () => test("internal binary protocol content-types stay centralized in transport constants", () => { const files = [ - ...jsFiles("shared"), - ...jsFiles("runtime"), - ...jsFiles("d1-runtime"), - ...jsFiles("do-runtime"), + ...PRODUCTION_JS_FILES, ...jsFiles("tests/unit"), ...jsFiles("tests/integration"), + ...rustFiles("rust"), ]; const literals = [ "application/vnd.wdl.d1-query", @@ -2191,6 +2453,7 @@ test("internal binary protocol content-types stay centralized in transport const "d1-runtime/protocol.js", "runtime/_wdl-do-transport.js", "do-runtime/protocol.js", + "rust/redis-proxy/src/runtime.rs", "tests/unit/runtime-load.test.js", "tests/unit/style-contracts.test.js", ]); @@ -2224,15 +2487,6 @@ test("internal binary protocol content-types stay centralized in transport const test("DO owner hints are trusted only from do-runtime headers", () => { const runtime = readRepoFile("runtime/_wdl-do-transport.js"); const doRuntime = readRepoFile("do-runtime/index.js"); - const match = runtime.match( - /export async function ownerHintFromResponse\(response\) \{[^]*?\n}\n\n(?:\/\*\*[^]*?\*\/\n)?export async function followOwnerHint/ - ); - assert.ok(match, "runtime DO transport must keep ownerHintFromResponse next to followOwnerHint"); - const ownerHintFromResponse = match[0]; - - assert.match(ownerHintFromResponse, /ownerHintFromHeaders\(response\.headers\)/); - assert.doesNotMatch(ownerHintFromResponse, /response\.clone\(\)\.json\(\)/); - assert.doesNotMatch(ownerHintFromResponse, /\.json\(\)/); const wrapper = doRuntime.match(/function withOwnerHintHeaders\(response, owner\) \{[^]*?\n}\n\n/)?.[0] || ""; assert.match(wrapper, /headers\.delete\(DO_OWNER_HINT_CONTROL_HEADER\)/); @@ -2245,34 +2499,64 @@ test("DO owner hints are trusted only from do-runtime headers", () => { assert.match(ownerHintHeadersFn, /\[DO_OWNER_HINT_HEADERS\.endpoint\]/); assert.match(ownerHintHeadersFn, /\[DO_OWNER_HINT_HEADERS\.generation\]/); assert.match(ownerHintHeadersFn, /if \(control\) headers\[DO_OWNER_HINT_CONTROL_HEADER\] = "1";/); + + assert.doesNotMatch(runtime, /export const OWNER_(?:HINT_STALE|RACE_RETRY)_CODES/); + assert.match(runtime, /const code = responseHeader\(response, DO_OWNERSHIP_ERROR_CONTROL_HEADER\)/); + assert.match(runtime, /setHas\(OWNER_RACE_RETRY_CODES, code\)/); + assert.match(doRuntime, /doPlatformErrorResponse\(err\)/); }); -test("owner endpoint validation lives in neutral runtime helper", () => { - const endpoint = readRepoFile("runtime/_wdl-owner-endpoint.js"); +test("owner endpoint validation lives in a shared contract owner", () => { + const endpoint = readRepoFile("shared/owner-endpoint.js"); + const adapter = readRepoFile("runtime/_wdl-owner-endpoint.js"); const doTransport = readRepoFile("runtime/_wdl-do-transport.js"); const d1Binding = readRepoFile("runtime/bindings/d1.js"); const controlD1RuntimeClient = readRepoFile("control/d1-runtime-client.js"); const userConfig = readRepoFile("runtime/config-user.capnp"); const systemConfig = readRepoFile("runtime/config-system.capnp"); + const d1Config = readRepoFile("d1-runtime/config.capnp"); const doConfig = readRepoFile("do-runtime/config.capnp"); + const taskIdentity = readRepoFile("shared/task-identity.js"); const tsconfig = readRepoFile("tsconfig.json"); assert.match(endpoint, /export function validOwnerEndpointForService/); assert.match(endpoint, /"d1-runtime": \/\^d1-runtime/); assert.match(endpoint, /"do-runtime": \/\^do-runtime/); + assert.match(endpoint, /function acceptablePrivateIpv4/); + assert.match(adapter, /from "\.\.\/shared\/owner-endpoint\.js"/); assert.match(doTransport, /from "\.\/_wdl-owner-endpoint\.js"/); - assert.match(d1Binding, /from "runtime-owner-endpoint"/); - assert.match(controlD1RuntimeClient, /from "runtime-owner-endpoint"/); + assert.match(d1Binding, /from "shared-owner-endpoint"/); + assert.match(controlD1RuntimeClient, /from "shared-owner-endpoint"/); assert.match(controlD1RuntimeClient, /validOwnerEndpointForService\(owner\.endpoint, 8787, "d1-runtime"\)/); - assert.match(tsconfig, /"runtime-owner-endpoint": \["runtime\/_wdl-owner-endpoint\.js"\]/); + assert.match(tsconfig, /"shared-\*": \["shared\/\*\.js"\]/); + assert.doesNotMatch(taskIdentity, /_TASK_PORT/); + assert.doesNotMatch(d1Config, /D1_TASK_PORT/); + assert.doesNotMatch(doConfig, /DO_TASK_PORT/); for (const config of [userConfig, systemConfig, doConfig]) { - assert.match(config, /name = "runtime-owner-endpoint"/); + assert.match(config, /name = "shared-owner-endpoint"/); assert.match(config, /name = "_wdl-owner-endpoint\.js"/); assert.match(config, /name = "runtime-owner-endpoint-source"/); } for (const config of [userConfig, systemConfig]) { const doOwnerNetwork = /const doOwnerNetworkWorker[\s\S]*?\n\);/.exec(config)?.[0] || ""; - assert.match(doOwnerNetwork, /name = "runtime-owner-endpoint"/); + assert.match(doOwnerNetwork, /name = "shared-owner-endpoint"/); + } +}); + +test("DO transport relative dependencies are registered in host and loaded-worker module graphs", () => { + const codeBudget = readRepoFile("runtime/load/code-budget.js"); + assert.match(codeBudget, /\["_wdl-request-id\.js", sources\.requestIdSource\]/); + assert.match(codeBudget, /\["_wdl-do-transport\.js", sources\.doTransportSource\]/); + + for (const file of [ + "runtime/config-user.capnp", + "runtime/config-system.capnp", + "do-runtime/config.capnp", + ]) { + const source = readRepoFile(file); + assert.match(source, /name = "runtime-do-transport", esModule = embed/); + assert.match(source, /name = "_wdl-request-id\.js", esModule = embed/); + assert.match(source, /name = "runtime-request-id-source", text = embed/); } }); @@ -2301,6 +2585,7 @@ test("queue delay cap literals stay aligned across standalone tiers", () => { test("Docker images build from pinned public base images", () => { const workerdDockerfile = withoutLineComments(readRepoFile("Dockerfile.workerd")); const rustDockerfile = withoutLineComments(readRepoFile("Dockerfile.rust")); + const cargoWorkspace = withoutLineComments(readRepoFile("rust/Cargo.toml")); const integrationEnvironment = withoutLineComments(readRepoFile("scripts/integration-environment.js")); assert.match(workerdDockerfile, /FROM rust:1-alpine AS supervisor-build/); @@ -2317,6 +2602,54 @@ test("Docker images build from pinned public base images", () => { assert.doesNotMatch(workerdDockerfile, /\bwget\b/); assert.doesNotMatch(workerdDockerfile, /^FROM\s+\S+:latest\b/m); assert.match(integrationEnvironment, /DOCKER_COMPOSE_BUILD_ARGS = \["compose", "build", "gateway", "workflows"\]/); + + const workspaceMembersMatch = /\bmembers\s*=\s*\[([\s\S]*?)\]/.exec(cargoWorkspace); + assert.ok(workspaceMembersMatch, "rust/Cargo.toml must declare workspace members"); + const workspaceMembers = [...workspaceMembersMatch[1].matchAll(/"([^"]+)"/g)] + .map((match) => match[1]) + .toSorted(); + const copiedManifests = (/** @type {string} */ source) => { + const copies = [...source.matchAll(/^COPY rust\/([^/\s]+)\/Cargo\.toml \.\/([^/\s]+)\/$/gm)]; + for (const copy of copies) { + assert.equal(copy[2], copy[1], `Cargo manifest destination must preserve ${copy[1]}`); + } + return copies.map((copy) => copy[1]).toSorted(); + }; + assert.deepEqual(copiedManifests(rustDockerfile), workspaceMembers); + assert.deepEqual(copiedManifests(workerdDockerfile), workspaceMembers); + + const imageLabels = (/** @type {string} */ source) => Object.fromEntries( + [...source.matchAll(/org\.opencontainers\.image\.([a-z]+)="([^"]*)"/g)] + .map((match) => [match[1], match[2]]) + ); + const rustLabels = imageLabels(rustDockerfile); + const workerdLabels = imageLabels(workerdDockerfile); + const sharedLabelKeys = [ + "source", + "documentation", + "vendor", + "authors", + "licenses", + "version", + "revision", + "created", + ]; + assert.deepEqual( + Object.fromEntries(sharedLabelKeys.map((key) => [key, rustLabels[key]])), + Object.fromEntries(sharedLabelKeys.map((key) => [key, workerdLabels[key]])) + ); + assert.notEqual(rustLabels.title, workerdLabels.title); + assert.notEqual(rustLabels.description, workerdLabels.description); + + for (const source of [rustDockerfile, workerdDockerfile]) { + for (const arg of ["WDL_VERSION", "VCS_REF", "BUILD_DATE"]) { + assert.equal( + [...source.matchAll(new RegExp(`^ARG ${arg}(?:=[^\\n]+)?$`, "gm"))].length, + 2, + `${arg} must be declared globally and in the final image stage` + ); + } + } }); test("workerd deploy configs use the supervisor-owned health check binary", () => { diff --git a/tests/unit/test-helper-style-contracts.test.js b/tests/unit/test-helper-style-contracts.test.js index 68ed4cf..05168d3 100644 --- a/tests/unit/test-helper-style-contracts.test.js +++ b/tests/unit/test-helper-style-contracts.test.js @@ -4,10 +4,26 @@ import { readdirSync } from "node:fs"; import { readRepoFile, repoPath, withoutLineComments } from "../helpers/source-scan.js"; import { + extractCapnpConstBlock, scannedTestFiles, + stripCapnpComments, withoutStringAndTemplateLiterals, } from "../helpers/style-contract-scanner.js"; +test("Cap'n Proto comment stripping preserves hashes inside strings", () => { + const source = ` +const demo :Config = ( + url = "https://example.test/path#fragment", + nested = (value = "escaped \\"# hash"), # trailing comment +); # final comment +`; + const stripped = stripCapnpComments(source); + assert.match(stripped, /path#fragment/); + assert.match(stripped, /escaped \\"# hash/); + assert.doesNotMatch(stripped, /trailing comment|final comment/); + assert.match(extractCapnpConstBlock(source, "demo"), /path#fragment/); +}); + test("test files use moduleDataUrl/repositoryFileUrl instead of raw fs+URL boilerplate", () => { // Skips: // test-helper-style-contracts.test.js — holds tripwire literals as error-message text. @@ -34,11 +50,7 @@ test("test files use moduleDataUrl/repositoryFileUrl instead of raw fs+URL boile }); test("test module source rewrites go through load-shared-module helpers", () => { - const EXEMPT = new Set([ - "tests/helpers/load-shared-module.js", - "tests/unit/test-helper-style-contracts.test.js", - ]); - const testFiles = scannedTestFiles(EXEMPT); + const testFiles = scannedTestFiles(); /** @type {string[]} */ const offenders = []; const sourceReader = String.raw`(?:readFileSync|readRepositoryFile|readRepositoryModuleSource)`; @@ -87,8 +99,7 @@ test("test module source rewrites go through load-shared-module helpers", () => }); test("test global mocks go through mock-global helpers", () => { - const EXEMPT = new Set(["tests/helpers/mock-global.js"]); - const testFiles = scannedTestFiles(EXEMPT); + const testFiles = scannedTestFiles(); /** @type {string[]} */ const offenders = []; for (const file of testFiles) { @@ -115,7 +126,6 @@ test("test global mocks go through mock-global helpers", () => { test("auth index tests access mock state through the harness", () => { const EXEMPT = new Set([ "tests/helpers/load-auth-index.js", - "tests/unit/test-helper-style-contracts.test.js", ]); const testFiles = scannedTestFiles(EXEMPT); /** @type {string[]} */ @@ -132,8 +142,6 @@ test("auth index tests access mock state through the harness", () => { test("test output capture goes through output-capture helpers", () => { const EXEMPT = new Set([ "tests/helpers/output-capture.js", - "tests/unit/output-capture.test.js", - "tests/unit/test-helper-style-contracts.test.js", ]); const testFiles = scannedTestFiles(EXEMPT); /** @type {string[]} */ @@ -153,9 +161,7 @@ test("test output capture goes through output-capture helpers", () => { test("test request-body and DO envelope decoders use shared helpers", () => { const EXEMPT = new Set([ - "tests/helpers/request-body.js", "tests/helpers/do-envelope.js", - "tests/unit/test-helper-style-contracts.test.js", ]); const testFiles = scannedTestFiles(EXEMPT); /** @type {string[]} */ @@ -181,11 +187,7 @@ test("test request-body and DO envelope decoders use shared helpers", () => { }); test("integration response JSON parsing uses response accessors", () => { - const EXEMPT = new Set([ - "tests/integration/helpers/http-response.js", - "tests/unit/test-helper-style-contracts.test.js", - ]); - const testFiles = scannedTestFiles(EXEMPT) + const testFiles = scannedTestFiles() .filter((file) => file.startsWith("tests/integration/")); /** @type {string[]} */ const offenders = []; @@ -284,11 +286,7 @@ test("integration Redis commands use typed Redis helpers", () => { }); test("integration status assertions use status helpers for structured diagnostics", () => { - const EXEMPT = new Set([ - "tests/integration/helpers/assertions.js", - "tests/unit/test-helper-style-contracts.test.js", - ]); - const testFiles = scannedTestFiles(EXEMPT) + const testFiles = scannedTestFiles() .filter((file) => file.startsWith("tests/integration/")); /** @type {string[]} */ const offenders = []; @@ -304,13 +302,7 @@ test("integration status assertions use status helpers for structured diagnostic }); test("unit JSON response assertions use response-json helper", () => { - const EXEMPT = new Set([ - "tests/helpers/response-json.js", - "tests/unit/mock-fetch.test.js", - "tests/unit/response-json.test.js", - "tests/unit/test-helper-style-contracts.test.js", - ]); - const testFiles = scannedTestFiles(EXEMPT) + const testFiles = scannedTestFiles() .filter((file) => file.startsWith("tests/unit/")); /** @type {string[]} */ const offenders = []; @@ -343,12 +335,7 @@ test("unit JSON response assertions use response-json helper", () => { }); test("tests use delay helper for simple sleep promises", () => { - const EXEMPT = new Set([ - "tests/helpers/timing.js", - "tests/integration/helpers/stack.js", - "tests/unit/test-helper-style-contracts.test.js", - ]); - const testFiles = scannedTestFiles(EXEMPT) + const testFiles = scannedTestFiles() .filter((file) => !file.includes("/manual/")); /** @type {string[]} */ const offenders = []; @@ -366,7 +353,6 @@ test("tests use delay helper for simple sleep promises", () => { test("test temporary directories use temp-dir helper", () => { const EXEMPT = new Set([ "tests/helpers/temp-dir.js", - "tests/unit/test-helper-style-contracts.test.js", ]); const testFiles = scannedTestFiles(EXEMPT); /** @type {string[]} */ @@ -386,9 +372,7 @@ test("test temporary directories use temp-dir helper", () => { test("DO owner hint test fixtures use shared helper", () => { const EXEMPT = new Set([ - "tests/helpers/do-owner-hint.js", "tests/unit/style-contracts.test.js", - "tests/unit/test-helper-style-contracts.test.js", ]); const testFiles = scannedTestFiles(EXEMPT); /** @type {string[]} */ @@ -404,11 +388,7 @@ test("DO owner hint test fixtures use shared helper", () => { }); test("repository JSON fixtures use readRepositoryJson", () => { - const EXEMPT = new Set([ - "tests/helpers/load-shared-module.js", - "tests/unit/test-helper-style-contracts.test.js", - ]); - const testFiles = scannedTestFiles(EXEMPT); + const testFiles = scannedTestFiles(); /** @type {string[]} */ const offenders = []; const fixtureJsonParse = /JSON\.parse\(\s*readFileSync\(\s*new URL\([^)]*fixtures\/[^)]*\.json/; @@ -422,15 +402,7 @@ test("repository JSON fixtures use readRepositoryJson", () => { }); test("structured JSON payloads use protocol-specific helpers", () => { - const EXEMPT = new Set([ - "tests/integration/helpers/http-response.js", - "tests/integration/helpers/internal-http.js", - "tests/integration/helpers/json-payload.js", - "tests/helpers/json-payload.js", - "tests/integration/helpers/ws-roundtrip-runner.cjs", - "tests/unit/test-helper-style-contracts.test.js", - ]); - const testFiles = scannedTestFiles(EXEMPT); + const testFiles = scannedTestFiles(); /** @type {string[]} */ const offenders = []; const structuredJsonParsers = [ diff --git a/tests/unit/version.test.js b/tests/unit/version.test.js index c263668..8db377f 100644 --- a/tests/unit/version.test.js +++ b/tests/unit/version.test.js @@ -1,6 +1,33 @@ import { test } from "node:test"; import assert from "node:assert/strict"; -import { formatVersion, parseVersion, bundleKey } from "../../shared/version.js"; +import { readRepositoryJson } from "../helpers/load-shared-module.js"; +import { + DECLARED_HOSTS_KEY, + DO_OWNER_SCOPE_PREFIX, + HOST_DECLARATIONS_SCAN_PATTERN, + HOSTS_SCAN_PATTERN, + NAMESPACES_KEY, + PATTERNS_CHANNEL, + ROUTES_CHANNEL, + ROUTES_FLUSH_CHANNEL, + VERSION_DELETE_LOCK_KIND, + WHOLE_DELETE_LOCK_KIND, + bundleKey, + deleteLockKey, + doOwnerScopeScanPatternForStorage, + doStorageIdKey, + formatDeleteLockToken, + formatVersion, + hostDeclarationsKey, + hostsKey, + namespaceFromHostsKey, + nextVersionKey, + nsHostsKey, + parseDeleteLockKind, + parseVersion, +} from "../../shared/version.js"; + +const versionFixture = readRepositoryJson("tests/fixtures/version-tags.json"); test("formatVersion: integer → v", () => { assert.equal(formatVersion(1), "v1"); @@ -32,6 +59,12 @@ test("parseVersion: returns null for malformed", () => { assert.equal(parseVersion(1), null); }); +test("parseVersion matches the shared JS/Rust version fixture", () => { + for (const { tag, parsed } of versionFixture.cases) { + assert.equal(parseVersion(tag), parsed, tag); + } +}); + test("bundleKey: composes worker:::v:", () => { assert.equal(bundleKey("demo", "hello", "v1"), "worker:demo:hello:v:1"); assert.equal(bundleKey("demo", "hello", "v42"), "worker:demo:hello:v:42"); @@ -43,3 +76,48 @@ test("bundleKey: rejects malformed version tags", () => { assert.throws(() => bundleKey("demo", "hello", ""), /invalid version/); assert.throws(() => bundleKey("demo", "hello", null), /invalid version/); }); + +test("route-plane registry key helpers compose and parse canonical keys", () => { + assert.equal(NAMESPACES_KEY, "namespaces"); + assert.equal(DECLARED_HOSTS_KEY, "declared-hosts"); + assert.equal(HOSTS_SCAN_PATTERN, "hosts:*"); + assert.equal(HOST_DECLARATIONS_SCAN_PATTERN, "host-declarations:*"); + assert.equal(hostsKey("demo"), "hosts:demo"); + assert.equal(namespaceFromHostsKey("hosts:demo"), "demo"); + assert.equal(namespaceFromHostsKey("routes:demo"), ""); + assert.equal(nsHostsKey("demo"), "ns-hosts:demo"); + assert.equal(hostDeclarationsKey("app.example"), "host-declarations:app.example"); + assert.equal(ROUTES_CHANNEL, "routes:invalidate"); + assert.equal(ROUTES_FLUSH_CHANNEL, "routes:flush"); + assert.equal(PATTERNS_CHANNEL, "patterns:invalidate"); +}); + +test("nextVersionKey composes the worker version counter key", () => { + assert.equal(nextVersionKey("demo", "hello"), "worker:demo:hello:next_version"); +}); + +test("worker lifecycle key helpers compose canonical keys", () => { + assert.equal(doStorageIdKey("demo", "hello"), "worker:do-storage:demo:hello"); + assert.equal(DO_OWNER_SCOPE_PREFIX, "do:owner:scope:"); + assert.equal( + doOwnerScopeScanPatternForStorage("do_abc"), + "do:owner:scope:do_abc%3A*" + ); + assert.equal(deleteLockKey("demo", "hello"), "worker-delete-lock:demo:hello"); +}); + +test("worker delete lock tokens carry the operation kind", () => { + assert.equal( + formatDeleteLockToken(WHOLE_DELETE_LOCK_KIND, "abc123"), + "whole:abc123" + ); + assert.equal( + formatDeleteLockToken(VERSION_DELETE_LOCK_KIND, "abc123"), + "version:abc123" + ); + assert.equal(parseDeleteLockKind("whole:abc123"), WHOLE_DELETE_LOCK_KIND); + assert.equal(parseDeleteLockKind("version:abc123"), VERSION_DELETE_LOCK_KIND); + assert.equal(parseDeleteLockKind("unknown:abc123"), null); + assert.equal(parseDeleteLockKind("whole:"), null); + assert.equal(parseDeleteLockKind("legacy-token"), null); +}); diff --git a/tsconfig.json b/tsconfig.json index 8447adc..b461fab 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -22,7 +22,6 @@ "runtime-dispatch-workflow-*": ["runtime/dispatch/workflow-*.js"], "runtime-load-*": ["runtime/load/*.js"], "runtime-do-transport": ["runtime/_wdl-do-transport.js"], - "runtime-owner-endpoint": ["runtime/_wdl-owner-endpoint.js"], "runtime-owner-hint-cache": ["runtime/_wdl-owner-hint-cache.js"], "runtime-state": ["runtime/runtime.js"], "runtime-*": ["runtime/*.js"],